backup_mongo_s3 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +24 -0
- data/Gemfile +3 -0
- data/README.md +17 -0
- data/backup_mongo_s3.gemspec +17 -0
- data/bin/backup_mongo_s3 +11 -0
- data/lib/backup_mongo_s3/application.rb +284 -0
- data/lib/backup_mongo_s3/db.rb +45 -0
- data/lib/backup_mongo_s3/scheduler.rb +102 -0
- data/lib/backup_mongo_s3/storage.rb +110 -0
- data/lib/backup_mongo_s3.rb +28 -0
- data/lib/helpers/config.yml.template +23 -0
- data/lib/helpers/fixnum.rb +11 -0
- data/lib/helpers/hash.rb +26 -0
- data/lib/helpers/time.rb +25 -0
- metadata +73 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 6818188f3168c9c81bb4872cb8879696dd6bc659
|
4
|
+
data.tar.gz: 7e64a7949927fc492e5f37b7f1da076bfbda157b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b37923f6e3c23e4b62322db449e8452989f6488b22598aa7c5497301ef018d8c2bb92659783e77fb93f9468287b37801def3de49723de1ee2949bdc3a6256909
|
7
|
+
data.tar.gz: c1af757aae23d51381d98d5071fc440e7fd07845dd324958e407a306b2ee66e180170ff45509fd14a7b18683ff7a536e6eca59b3a7b52e383a0684121bef6406
|
data/.gitignore
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
*.idea
|
4
|
+
.bundle
|
5
|
+
.config
|
6
|
+
.yardoc
|
7
|
+
Gemfile.lock
|
8
|
+
InstalledFiles
|
9
|
+
_yardoc
|
10
|
+
coverage
|
11
|
+
doc/
|
12
|
+
lib/bundler/man
|
13
|
+
pkg
|
14
|
+
rdoc
|
15
|
+
spec/reports
|
16
|
+
test/tmp
|
17
|
+
test/version_tmp
|
18
|
+
tmp
|
19
|
+
*.bundle
|
20
|
+
*.so
|
21
|
+
*.o
|
22
|
+
*.a
|
23
|
+
mkmf.log
|
24
|
+
*.yml
|
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
# BackupMongoS3
|
2
|
+
|
3
|
+
TODO: Write a gem description
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'backup_mongo_s3'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install backup_mongo_s3
|
@@ -0,0 +1,17 @@
|
|
1
|
+
Gem::Specification.new do |spec|
|
2
|
+
spec.name = 'backup_mongo_s3'
|
3
|
+
spec.version = '0.0.2'
|
4
|
+
spec.authors = ['Yakupov Dima']
|
5
|
+
spec.email = ['yakupov.dima@mail.ru']
|
6
|
+
spec.summary = "Some summary"
|
7
|
+
spec.description = "Some description"
|
8
|
+
spec.homepage = ''
|
9
|
+
spec.license = 'MIT'
|
10
|
+
|
11
|
+
spec.files = `git ls-files -z`.split("\x0")
|
12
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
13
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
14
|
+
spec.require_paths = ["lib"]
|
15
|
+
|
16
|
+
spec.add_development_dependency 'bundler', '~> 1.6'
|
17
|
+
end
|
data/bin/backup_mongo_s3
ADDED
@@ -0,0 +1,284 @@
|
|
1
|
+
module BackupMongoS3
|
2
|
+
class Application
|
3
|
+
|
4
|
+
def initialize(argv)
|
5
|
+
@parser = OptionParser.new
|
6
|
+
|
7
|
+
@params = parse_options(argv)
|
8
|
+
@config = parse_config(@params[:config])
|
9
|
+
|
10
|
+
@db = Db.new(@config[:mongo])
|
11
|
+
@storage = Storage.new(@config[:s3])
|
12
|
+
end
|
13
|
+
|
14
|
+
public
|
15
|
+
|
16
|
+
def run
|
17
|
+
|
18
|
+
case
|
19
|
+
when @params[:backups_list] # BACKUPS_LIST
|
20
|
+
show_backups_list(@params[:backups_list])
|
21
|
+
|
22
|
+
when @params[:backup_all] # BACKUP_ALL
|
23
|
+
dbs_str = @config[:backup][:dbs]
|
24
|
+
|
25
|
+
if dbs_str.nil? || dbs_str.empty?
|
26
|
+
raise 'config.yml::backup.dbs is empty'
|
27
|
+
end
|
28
|
+
|
29
|
+
dbs_name = dbs_str.split(',').each { |db_name| db_name.strip! }
|
30
|
+
|
31
|
+
backup(dbs_name)
|
32
|
+
|
33
|
+
when @params[:backup] # BACKUP
|
34
|
+
backup([@params[:backup]])
|
35
|
+
|
36
|
+
when @params[:restore] # RESTORE
|
37
|
+
if @params[:backup_date].nil?
|
38
|
+
raise 'param --date BACKUP_DATE is not specified'
|
39
|
+
end
|
40
|
+
|
41
|
+
restore(@params[:restore], @params[:backup_date])
|
42
|
+
|
43
|
+
when @params[:cron_update] || @params[:cron_clear] # CRON
|
44
|
+
cron_options =
|
45
|
+
{
|
46
|
+
update: @params[:cron_update],
|
47
|
+
clear: @params[:cron_clear],
|
48
|
+
config: @params[:config],
|
49
|
+
time: @config[:backup][:cron_time],
|
50
|
+
}
|
51
|
+
|
52
|
+
Scheduler.new(cron_options).execute
|
53
|
+
|
54
|
+
else
|
55
|
+
puts "\n#{@parser}\n"
|
56
|
+
exit
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def show_backups_list(db_name)
|
64
|
+
|
65
|
+
backups = @storage.get_backups_list("#{db_name}")
|
66
|
+
|
67
|
+
if backups.empty?
|
68
|
+
puts 'Backups not found'
|
69
|
+
else
|
70
|
+
|
71
|
+
puts sprintf('%-30s %-15s %-20s', 'name', 'size, MB', 'last_modified')
|
72
|
+
|
73
|
+
backups.each do |backup|
|
74
|
+
|
75
|
+
backup_name = File.join(File.dirname(backup.key), File.basename(backup.key, '.backup'))
|
76
|
+
backup_size = backup.content_length / 1024 / 1024
|
77
|
+
backup_last_modified = backup.last_modified
|
78
|
+
|
79
|
+
puts sprintf('%-30s %-15s %-20s', backup_name, backup_size, backup_last_modified)
|
80
|
+
end
|
81
|
+
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def backup(dbs_name)
|
86
|
+
|
87
|
+
history_days_str = @config[:backup][:history_days]
|
88
|
+
|
89
|
+
begin
|
90
|
+
history_days = history_days_str.to_i
|
91
|
+
rescue
|
92
|
+
raise 'config.yml::backup.history_days is not integer'
|
93
|
+
end
|
94
|
+
|
95
|
+
dbs_name.each do |db_name|
|
96
|
+
|
97
|
+
puts "backup db #{db_name.upcase}:"
|
98
|
+
|
99
|
+
tmp_dir = get_temp_dir
|
100
|
+
|
101
|
+
begin
|
102
|
+
|
103
|
+
dump_path = File.join(tmp_dir, db_name)
|
104
|
+
|
105
|
+
puts "\t dump db..."
|
106
|
+
@db.dump(db_name, tmp_dir)
|
107
|
+
|
108
|
+
if Dir["#{dump_path}/*"].empty?
|
109
|
+
puts "\t [skip] db is empty"
|
110
|
+
|
111
|
+
else
|
112
|
+
|
113
|
+
puts "\t compress..."
|
114
|
+
system("zip -6 -r '#{dump_path}.zip' '#{dump_path}/' -j > /dev/null")
|
115
|
+
raise 'Error zip' unless $?.exitstatus.zero?
|
116
|
+
|
117
|
+
FileUtils.rm_rf(dump_path)
|
118
|
+
|
119
|
+
zip_file_path = "#{dump_path}.zip"
|
120
|
+
|
121
|
+
if File.exists?(zip_file_path)
|
122
|
+
puts "\t upload backup to s3..."
|
123
|
+
@storage.upload(db_name, zip_file_path)
|
124
|
+
|
125
|
+
puts "\t delete old backups from s3..."
|
126
|
+
@storage.delete_old_backups(db_name, history_days)
|
127
|
+
|
128
|
+
File.delete(zip_file_path)
|
129
|
+
end
|
130
|
+
|
131
|
+
end
|
132
|
+
|
133
|
+
ensure
|
134
|
+
FileUtils.remove_entry_secure(tmp_dir)
|
135
|
+
end
|
136
|
+
|
137
|
+
end
|
138
|
+
|
139
|
+
puts '[done] backup'
|
140
|
+
end
|
141
|
+
|
142
|
+
|
143
|
+
def restore(db_name, date)
|
144
|
+
|
145
|
+
puts "restore db #{db_name.upcase}:"
|
146
|
+
|
147
|
+
tmp_dir = get_temp_dir
|
148
|
+
|
149
|
+
begin
|
150
|
+
|
151
|
+
dump_path = File.join(tmp_dir, db_name)
|
152
|
+
|
153
|
+
zip_file_path = "#{dump_path}.zip"
|
154
|
+
|
155
|
+
puts "\t download file from s3..."
|
156
|
+
@storage.download(db_name, date, zip_file_path)
|
157
|
+
|
158
|
+
if File.exists?(zip_file_path)
|
159
|
+
puts "\t uncompress..."
|
160
|
+
system("unzip '#{zip_file_path}' -d '#{dump_path}' > /dev/null")
|
161
|
+
raise 'Error unzip' unless $?.exitstatus.zero?
|
162
|
+
|
163
|
+
File.delete(zip_file_path)
|
164
|
+
|
165
|
+
puts "\t restore db..."
|
166
|
+
@db.restore(db_name, dump_path)
|
167
|
+
end
|
168
|
+
|
169
|
+
ensure
|
170
|
+
FileUtils.remove_entry_secure(tmp_dir)
|
171
|
+
end
|
172
|
+
|
173
|
+
puts '[done] restore'
|
174
|
+
end
|
175
|
+
|
176
|
+
def create_config(path)
|
177
|
+
|
178
|
+
path = '.' if path.nil? || path == ''
|
179
|
+
|
180
|
+
file = File.join(path, 'config.yml')
|
181
|
+
|
182
|
+
if File.exists?(file)
|
183
|
+
raise "create_config: '#{file}' already exists"
|
184
|
+
elsif File.exists?(file.downcase)
|
185
|
+
raise "create_config: '#{file.downcase}' exists, which could conflict with '#{file}'"
|
186
|
+
elsif !File.exists?(File.dirname(file))
|
187
|
+
raise "create_config: directory '#{File.dirname(file)}' does not exist"
|
188
|
+
else
|
189
|
+
file_template = File.join(BackupMongoS3.root_path, 'lib/helpers/config.yml.template')
|
190
|
+
|
191
|
+
FileUtils.cp file_template, file
|
192
|
+
end
|
193
|
+
|
194
|
+
puts "[done] file #{file} was created"
|
195
|
+
end
|
196
|
+
|
197
|
+
def parse_options(argv)
|
198
|
+
params = {}
|
199
|
+
|
200
|
+
@parser.on('--backup_all', 'Backup databases specified in config.yml and upload to S3 bucket') do
|
201
|
+
params[:backup_all] = true
|
202
|
+
end
|
203
|
+
@parser.on('--backup DB_NAME', String, 'Backup database and upload to S3 bucket') do |db_name|
|
204
|
+
params[:backup] = db_name
|
205
|
+
end
|
206
|
+
@parser.on('-r', '--restore DB_NAME', String, 'Restore database from BACKUP_DATE backup') do |db_name|
|
207
|
+
params[:restore] = db_name
|
208
|
+
end
|
209
|
+
@parser.on('-d', '--date BACKUP_DATE', String, 'Restore date YYYYMMDD') do |backup_date|
|
210
|
+
params[:backup_date] = backup_date
|
211
|
+
end
|
212
|
+
@parser.on('-l', '--list_backups [DB_NAME]', String, 'Show list of available backups') do |db_name|
|
213
|
+
params[:backups_list] = db_name || ''
|
214
|
+
end
|
215
|
+
@parser.on('--write_cron', 'Add/update backup_all job') do
|
216
|
+
params[:cron_update] = true
|
217
|
+
end
|
218
|
+
@parser.on('--clear_cron', 'Clear backup_all job') do
|
219
|
+
params[:cron_clear] = true
|
220
|
+
end
|
221
|
+
@parser.on('-c', '--config PATH', String, 'Path to config *.yml. Default ./config.yml') do |path|
|
222
|
+
params[:config] = path || ''
|
223
|
+
end
|
224
|
+
@parser.on('--create_config [PATH]', String, 'Create template config.yml in current/PATH directory') do |path|
|
225
|
+
create_config(path)
|
226
|
+
exit
|
227
|
+
end
|
228
|
+
@parser.on('-h', '--help', 'Show help') do
|
229
|
+
puts "\n#{@parser}\n"
|
230
|
+
exit
|
231
|
+
end
|
232
|
+
|
233
|
+
begin
|
234
|
+
@parser.parse!(argv)
|
235
|
+
|
236
|
+
rescue OptionParser::ParseError => err
|
237
|
+
puts "#{err.message}\n\n#{@parser}"
|
238
|
+
exit
|
239
|
+
end
|
240
|
+
|
241
|
+
if [params[:backup_all], params[:backup], params[:restore], params[:backups_list], params[:cron_update], params[:cron_clear]].compact.length > 1
|
242
|
+
raise 'Can only backup_all, backup, restore, backups_list, cron_update or cron_clear. Choose one.'
|
243
|
+
end
|
244
|
+
|
245
|
+
if params[:config].nil? || params[:config] == ''
|
246
|
+
params[:config] = './config.yml'
|
247
|
+
end
|
248
|
+
|
249
|
+
params[:config] = File.absolute_path(params[:config])
|
250
|
+
|
251
|
+
params
|
252
|
+
end
|
253
|
+
|
254
|
+
def parse_config(config_path)
|
255
|
+
|
256
|
+
begin
|
257
|
+
config = YAML.load(File.read(config_path))
|
258
|
+
rescue Errno::ENOENT
|
259
|
+
raise "Could not find config file '#{config_path}'"
|
260
|
+
rescue ArgumentError => err
|
261
|
+
raise "Could not parse config file '#{config_path}' - #{err}"
|
262
|
+
end
|
263
|
+
|
264
|
+
config.deep_symbolize_keys!
|
265
|
+
|
266
|
+
raise 'config.yml. Section <backup> not found' if config[:backup].nil?
|
267
|
+
raise 'config.yml. Section <mongo> not found' if config[:mongo].nil?
|
268
|
+
raise 'config.yml. Section <s3> not found' if config[:s3].nil?
|
269
|
+
|
270
|
+
config
|
271
|
+
end
|
272
|
+
|
273
|
+
def get_temp_dir
|
274
|
+
temp_dir = @config[:backup][:temp_directory]
|
275
|
+
|
276
|
+
if temp_dir.nil? || temp_dir == ''
|
277
|
+
temp_dir = Dir.tmpdir
|
278
|
+
end
|
279
|
+
|
280
|
+
Dir.mktmpdir(nil, temp_dir)
|
281
|
+
end
|
282
|
+
|
283
|
+
end
|
284
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module BackupMongoS3
|
2
|
+
class Db
|
3
|
+
|
4
|
+
def initialize(options)
|
5
|
+
@connection_options = connection(options)
|
6
|
+
end
|
7
|
+
|
8
|
+
private
|
9
|
+
def connection(options)
|
10
|
+
|
11
|
+
host = (options[:host].nil? || options[:host].empty?) ? 'localhost' : options[:host]
|
12
|
+
port = options[:port].nil? ? 27017 : options[:port]
|
13
|
+
username = options[:username]
|
14
|
+
password = options[:password]
|
15
|
+
|
16
|
+
auth_options = ''
|
17
|
+
|
18
|
+
unless username.nil? || username.empty? || password.nil? || password.empty?
|
19
|
+
auth_options = "-u '#{username}' -p '#{password}'"
|
20
|
+
end
|
21
|
+
|
22
|
+
"--host '#{host}' --port '#{port}' #{auth_options}"
|
23
|
+
end
|
24
|
+
|
25
|
+
public
|
26
|
+
def dump(db_name, backup_path)
|
27
|
+
command = "mongodump --dumpDbUsersAndRoles #{@connection_options} --db '#{db_name}' --out '#{backup_path}'"
|
28
|
+
command << ' > /dev/null'
|
29
|
+
|
30
|
+
system(command)
|
31
|
+
raise "Error mongodump '#{db_name}'" unless $?.exitstatus.zero?
|
32
|
+
end
|
33
|
+
|
34
|
+
public
|
35
|
+
def restore(db_name, backup_path)
|
36
|
+
command = "mongorestore --restoreDbUsersAndRoles #{@connection_options} --db '#{db_name}' '#{backup_path}'"
|
37
|
+
command << ' > /dev/null'
|
38
|
+
|
39
|
+
system(command)
|
40
|
+
raise "Error mongodump '#{db_name}'" unless $?.exitstatus.zero?
|
41
|
+
end
|
42
|
+
|
43
|
+
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
module BackupMongoS3
|
2
|
+
class Scheduler
|
3
|
+
|
4
|
+
def initialize(options = {})
|
5
|
+
@options = options
|
6
|
+
|
7
|
+
if [@options[:write], @options[:clear]].compact.length > 1
|
8
|
+
raise 'cron: Can only write or clear. Choose one.'
|
9
|
+
end
|
10
|
+
|
11
|
+
unless @options[:time] =~ /\A[*\-,0-9]+ [*\-,0-9]+ [*\-,0-9]+ [*\-,0-9]+ [*\-,0-6]+\z/
|
12
|
+
raise 'config.yml: cron_time is not valid'
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def execute
|
17
|
+
write_crontab(updated_crontab)
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def read_crontab
|
23
|
+
return @read_crontab if @read_crontab
|
24
|
+
|
25
|
+
command = 'crontab -l'
|
26
|
+
|
27
|
+
command_results = %x[#{command} 2> /dev/null]
|
28
|
+
|
29
|
+
@read_crontab = $?.exitstatus.zero? ? prepare(command_results) : ''
|
30
|
+
end
|
31
|
+
|
32
|
+
def prepare(contents)
|
33
|
+
# Some cron implementations require all non-comment lines to be newline-
|
34
|
+
# terminated. (issue #95) Strip all newlines and replace with the default
|
35
|
+
# platform record seperator ($/)
|
36
|
+
contents.gsub!(/\s+$/, $/)
|
37
|
+
end
|
38
|
+
|
39
|
+
def write_crontab(contents)
|
40
|
+
command = 'crontab -'
|
41
|
+
|
42
|
+
IO.popen(command, 'r+') do |crontab|
|
43
|
+
crontab.write(contents)
|
44
|
+
crontab.close_write
|
45
|
+
end
|
46
|
+
|
47
|
+
success = $?.exitstatus.zero?
|
48
|
+
|
49
|
+
if success
|
50
|
+
action = @options[:update] ? 'updated' : 'cleared'
|
51
|
+
puts "[done] crontab file #{action}"
|
52
|
+
exit(0)
|
53
|
+
else
|
54
|
+
raise "Couldn't write crontab"
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def updated_crontab
|
59
|
+
# Check for unopened or unclosed identifier blocks
|
60
|
+
if read_crontab =~ Regexp.new("^#{comment_open}\s*$") && (read_crontab =~ Regexp.new("^#{comment_close}\s*$")).nil?
|
61
|
+
raise "Unclosed indentifier; Your crontab file contains '#{comment_open}', but no '#{comment_close}'"
|
62
|
+
elsif (read_crontab =~ Regexp.new("^#{comment_open}\s*$")).nil? && read_crontab =~ Regexp.new("^#{comment_close}\s*$")
|
63
|
+
raise "Unopened indentifier; Your crontab file contains '#{comment_close}', but no '#{comment_open}'"
|
64
|
+
end
|
65
|
+
|
66
|
+
# If an existing identier block is found, replace it with the new cron entries
|
67
|
+
if read_crontab =~ Regexp.new("^#{comment_open}\s*$") && read_crontab =~ Regexp.new("^#{comment_close}\s*$")
|
68
|
+
# If the existing crontab file contains backslashes they get lost going through gsub.
|
69
|
+
# .gsub('\\', '\\\\\\') preserves them. Go figure.
|
70
|
+
read_crontab.gsub(Regexp.new("^#{comment_open}\s*$.+^#{comment_close}\s*$", Regexp::MULTILINE), crontab_task.chomp.gsub('\\', '\\\\\\'))
|
71
|
+
else # Otherwise, append the new cron entries after any existing ones
|
72
|
+
[read_crontab, crontab_task].join("\n\n")
|
73
|
+
end.gsub(/\n{3,}/, "\n\n") # More than two newlines becomes just two.
|
74
|
+
end
|
75
|
+
|
76
|
+
def crontab_task
|
77
|
+
return '' if @options[:clear]
|
78
|
+
[comment_open, crontab_job, comment_close].compact.join("\n") + "\n"
|
79
|
+
end
|
80
|
+
|
81
|
+
def crontab_job
|
82
|
+
job = [@options[:time]]
|
83
|
+
job << BackupMongoS3.name
|
84
|
+
job << '--backup_all'
|
85
|
+
job << "--config #{@options[:config]}"
|
86
|
+
|
87
|
+
job.join(' ')
|
88
|
+
end
|
89
|
+
|
90
|
+
def comment_base
|
91
|
+
"#{BackupMongoS3.name} task"
|
92
|
+
end
|
93
|
+
|
94
|
+
def comment_open
|
95
|
+
"# Begin #{comment_base}"
|
96
|
+
end
|
97
|
+
|
98
|
+
def comment_close
|
99
|
+
"# End #{comment_base}"
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
@@ -0,0 +1,110 @@
|
|
1
|
+
module BackupMongoS3
|
2
|
+
class Storage
|
3
|
+
|
4
|
+
def initialize(options)
|
5
|
+
@s3 = AWS::S3.new({access_key_id: options[:access_key_id], secret_access_key: options[:secret_access_key]})
|
6
|
+
@bucket = get_bucket(options[:bucket])
|
7
|
+
end
|
8
|
+
|
9
|
+
private
|
10
|
+
def get_bucket(bucket_name)
|
11
|
+
bucket = @s3.buckets[bucket_name]
|
12
|
+
|
13
|
+
unless bucket.exists?
|
14
|
+
raise "Bucket #{bucket_name} doesn't not exists. Please create it"
|
15
|
+
end
|
16
|
+
|
17
|
+
bucket
|
18
|
+
end
|
19
|
+
|
20
|
+
public
|
21
|
+
def upload(storage_path, file_name)
|
22
|
+
key = File.join(storage_path, Time.now.utc.strftime('%Y%m%d') + '.backup')
|
23
|
+
|
24
|
+
checksum = get_signature(file_name)
|
25
|
+
|
26
|
+
begin
|
27
|
+
|
28
|
+
file = File.open(file_name, 'rb')
|
29
|
+
|
30
|
+
obj = @bucket.objects[key]
|
31
|
+
|
32
|
+
obj.write(:content_length => file.size, metadata: {checksum: checksum}) do |buffer, bytes|
|
33
|
+
buffer.write(file.read(bytes))
|
34
|
+
end
|
35
|
+
|
36
|
+
file.close
|
37
|
+
|
38
|
+
rescue Exception => err
|
39
|
+
raise "Error upload file <#{file_name}> to s3 <#{key}>: #{err.message}"
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
|
44
|
+
public
|
45
|
+
def download(storage_path, storage_file_name, file_name)
|
46
|
+
key = File.join(storage_path, storage_file_name + '.backup')
|
47
|
+
|
48
|
+
begin
|
49
|
+
|
50
|
+
file = File.open(file_name, 'wb')
|
51
|
+
|
52
|
+
obj = @bucket.objects[key]
|
53
|
+
|
54
|
+
response = obj.read do |chunk|
|
55
|
+
file.write(chunk)
|
56
|
+
end
|
57
|
+
|
58
|
+
file.close
|
59
|
+
|
60
|
+
checksum = get_signature(file_name)
|
61
|
+
|
62
|
+
if checksum != response[:meta]['checksum']
|
63
|
+
raise 'Backup signature is not valid'
|
64
|
+
end
|
65
|
+
|
66
|
+
rescue Exception => err
|
67
|
+
raise "Error download file <#{key}> from s3 to <#{file_name}>: #{err.message}"
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
71
|
+
|
72
|
+
public
|
73
|
+
def get_backups_list(prefix = '', limit = 100)
|
74
|
+
result =[]
|
75
|
+
|
76
|
+
@bucket.objects.with_prefix(prefix).each(:limit => limit) do |object|
|
77
|
+
if File.extname(object.key) == '.backup'
|
78
|
+
result << object
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
result
|
83
|
+
end
|
84
|
+
|
85
|
+
public
|
86
|
+
def delete_old_backups(prefix, history_days)
|
87
|
+
|
88
|
+
old_backups = []
|
89
|
+
|
90
|
+
backups_time_limit = Time.now.utc.midnight - history_days.days
|
91
|
+
|
92
|
+
backups = get_backups_list(prefix)
|
93
|
+
|
94
|
+
backups.each do |backup|
|
95
|
+
if backup.last_modified.utc < backups_time_limit
|
96
|
+
old_backups << backup
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
@bucket.delete(old_backups) unless old_backups.empty?
|
101
|
+
|
102
|
+
end
|
103
|
+
|
104
|
+
private
|
105
|
+
def get_signature(file_name)
|
106
|
+
Digest::MD5.hexdigest(File.size(file_name).to_s)
|
107
|
+
end
|
108
|
+
|
109
|
+
end
|
110
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'aws-sdk'
|
3
|
+
require 'fileutils'
|
4
|
+
require 'digest/md5'
|
5
|
+
require 'tmpdir'
|
6
|
+
|
7
|
+
require_relative 'backup_mongo_s3/application'
|
8
|
+
require_relative 'backup_mongo_s3/db'
|
9
|
+
require_relative 'backup_mongo_s3/storage'
|
10
|
+
require_relative 'backup_mongo_s3/scheduler'
|
11
|
+
|
12
|
+
require_relative 'helpers/fixnum'
|
13
|
+
require_relative 'helpers/hash'
|
14
|
+
require_relative 'helpers/time'
|
15
|
+
|
16
|
+
module BackupMongoS3
|
17
|
+
|
18
|
+
def self.name
|
19
|
+
File.basename( __FILE__, '.rb')
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.root_path
|
23
|
+
@root_path if @root_path
|
24
|
+
spec = Gem::Specification.find_by_name(self.name)
|
25
|
+
@root_path = spec.gem_dir
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
@@ -0,0 +1,23 @@
|
|
1
|
+
backup:
|
2
|
+
# List dbs for backup when run backup_all
|
3
|
+
dbs: db1, db2
|
4
|
+
|
5
|
+
# All backups older than [today - history_days] will be deleted when run backup/backup_all
|
6
|
+
history_days: 5
|
7
|
+
|
8
|
+
# Temporary directory for dump files. Default system temp directory
|
9
|
+
temp_directory:
|
10
|
+
|
11
|
+
# [minute] [hour] [day] [month] [weekday]
|
12
|
+
cron_time: 0 0 * * *
|
13
|
+
|
14
|
+
mongo:
|
15
|
+
host: 'localhost'
|
16
|
+
port: 27017
|
17
|
+
username:
|
18
|
+
password:
|
19
|
+
|
20
|
+
s3:
|
21
|
+
access_key_id: 'access_key_id'
|
22
|
+
secret_access_key: 'secret_access_key'
|
23
|
+
bucket: 'backup_mongo_s3-backups'
|
data/lib/helpers/hash.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
class Hash
|
2
|
+
|
3
|
+
def symbolize_keys!
|
4
|
+
transform_keys! { |key| key.to_sym rescue key }
|
5
|
+
end
|
6
|
+
|
7
|
+
def deep_symbolize_keys!
|
8
|
+
deep_transform_keys!{ |key| key.to_sym rescue key }
|
9
|
+
end
|
10
|
+
|
11
|
+
def transform_keys!
|
12
|
+
keys.each do |key|
|
13
|
+
self[yield(key)] = delete(key)
|
14
|
+
end
|
15
|
+
self
|
16
|
+
end
|
17
|
+
|
18
|
+
def deep_transform_keys!(&block)
|
19
|
+
keys.each do |key|
|
20
|
+
value = delete(key)
|
21
|
+
self[yield(key)] = value.is_a?(Hash) ? value.deep_transform_keys!(&block) : value
|
22
|
+
end
|
23
|
+
self
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
data/lib/helpers/time.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
class Time
|
2
|
+
|
3
|
+
def midnight
|
4
|
+
change(:hour => 0)
|
5
|
+
end
|
6
|
+
|
7
|
+
def change(options)
|
8
|
+
new_year = options.fetch(:year, year)
|
9
|
+
new_month = options.fetch(:month, month)
|
10
|
+
new_day = options.fetch(:day, day)
|
11
|
+
new_hour = options.fetch(:hour, hour)
|
12
|
+
new_min = options.fetch(:min, options[:hour] ? 0 : min)
|
13
|
+
new_sec = options.fetch(:sec, (options[:hour] || options[:min]) ? 0 : sec)
|
14
|
+
new_usec = options.fetch(:usec, (options[:hour] || options[:min] || options[:sec]) ? 0 : Rational(nsec, 1000))
|
15
|
+
|
16
|
+
if utc?
|
17
|
+
::Time.utc(new_year, new_month, new_day, new_hour, new_min, new_sec, new_usec)
|
18
|
+
elsif zone
|
19
|
+
::Time.local(new_year, new_month, new_day, new_hour, new_min, new_sec, new_usec)
|
20
|
+
else
|
21
|
+
::Time.new(new_year, new_month, new_day, new_hour, new_min, new_sec + (new_usec.to_r / 1000000), utc_offset)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
metadata
ADDED
@@ -0,0 +1,73 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: backup_mongo_s3
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.2
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Yakupov Dima
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-10-30 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.6'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.6'
|
27
|
+
description: Some description
|
28
|
+
email:
|
29
|
+
- yakupov.dima@mail.ru
|
30
|
+
executables:
|
31
|
+
- backup_mongo_s3
|
32
|
+
extensions: []
|
33
|
+
extra_rdoc_files: []
|
34
|
+
files:
|
35
|
+
- .gitignore
|
36
|
+
- Gemfile
|
37
|
+
- README.md
|
38
|
+
- backup_mongo_s3.gemspec
|
39
|
+
- bin/backup_mongo_s3
|
40
|
+
- lib/backup_mongo_s3.rb
|
41
|
+
- lib/backup_mongo_s3/application.rb
|
42
|
+
- lib/backup_mongo_s3/db.rb
|
43
|
+
- lib/backup_mongo_s3/scheduler.rb
|
44
|
+
- lib/backup_mongo_s3/storage.rb
|
45
|
+
- lib/helpers/config.yml.template
|
46
|
+
- lib/helpers/fixnum.rb
|
47
|
+
- lib/helpers/hash.rb
|
48
|
+
- lib/helpers/time.rb
|
49
|
+
homepage: ''
|
50
|
+
licenses:
|
51
|
+
- MIT
|
52
|
+
metadata: {}
|
53
|
+
post_install_message:
|
54
|
+
rdoc_options: []
|
55
|
+
require_paths:
|
56
|
+
- lib
|
57
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - '>='
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: '0'
|
67
|
+
requirements: []
|
68
|
+
rubyforge_project:
|
69
|
+
rubygems_version: 2.4.2
|
70
|
+
signing_key:
|
71
|
+
specification_version: 4
|
72
|
+
summary: Some summary
|
73
|
+
test_files: []
|