ey_cloud_server 1.4.54 → 1.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/bin/ey-snapshots +2 -1
- data/bin/eybackup +4 -2
- data/bin/eyrestore +211 -0
- data/features/step_definitions/mysql.rb +2 -0
- data/features/support/env.rb +3 -0
- data/lib/ey-flex/bucket_minder.rb +11 -1
- data/lib/ey-flex.rb +1 -1
- data/lib/ey_backup/backend.rb +2 -3
- data/lib/ey_backup/backup_set.rb +16 -16
- data/lib/ey_backup/cli.rb +67 -10
- data/lib/ey_backup/dumper.rb +20 -10
- data/lib/ey_backup/engine.rb +16 -3
- data/lib/ey_backup/engines/mysql_engine.rb +16 -5
- data/lib/ey_backup/engines/postgresql_engine.rb +37 -12
- data/lib/ey_backup/loader.rb +3 -1
- data/lib/ey_backup/logger.rb +42 -10
- data/lib/ey_backup/processors/splitter.rb +1 -1
- data/lib/ey_backup/spawner.rb +2 -5
- data/lib/ey_backup.rb +13 -2
- data/lib/ey_cloud_server/version.rb +1 -1
- data/spec/config.yml.ci +11 -0
- data/spec/ey_backup/backup_spec.rb +2 -0
- data/spec/ey_backup/cli_spec.rb +2 -0
- data/spec/ey_backup/mysql_backups_spec.rb +26 -0
- data/spec/ey_backup/postgres_backups_spec.rb +12 -0
- data/spec/helpers.rb +40 -14
- data/spec/spec_helper.rb +3 -0
- metadata +33 -37
- data/spec/config.yml +0 -11
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
ZmFkNjBkYjY1OTNhZjQwNjMxNjUxMmNmOTRmOWQyZDMxMDJlNTUzYQ==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
YTA0NTBmMTAxYzk5ZmE5YjQ0MzViYjE4NWE2YjRjZGM5YWYxZDNiNQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
ZmRlZTQ1YThjMWU0N2RkMzVlNmFkNGY3YThmMDJhMzRjZTE4MzU4ZGJjOTAx
|
10
|
+
NzM5MDc4OGZmNTMxNzM1OGI1ZTk4NDgxZjY5MTIwYmZmMTI1NmNjNjc0NmFl
|
11
|
+
OTI3OWFmMjdiZGE0N2QwZmEyNjljYzQ0NTg0MjBmZWI3NTBiOWU=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
MTdlZDkwYmQ1ZmNiYTJmY2U0YWMzMmQ2YjZmNzA5NWEzYTVlNTA4ODM1ZGZj
|
14
|
+
ZTBkN2EwZmVlODczYTAxMjYwNzA3OTk3ODcxODlkOTQwZjYwZDNhNjE4Mjk2
|
15
|
+
YThjMTZiNjBmMmY3NTE5Yjg0NjI2YjRkOGQwMDg0YmQ4OTY3ZGM=
|
data/bin/ey-snapshots
CHANGED
data/bin/eybackup
CHANGED
@@ -12,7 +12,9 @@ require File.dirname(__FILE__) + '/../lib/ey-flex'
|
|
12
12
|
begin
|
13
13
|
EY::Backup.run(ARGV)
|
14
14
|
rescue => e
|
15
|
-
|
16
|
-
|
15
|
+
puts e.message
|
16
|
+
puts "******* Trace *******"
|
17
|
+
puts e.inspect
|
18
|
+
EY.notify_backup_error(e) unless STDOUT.isatty
|
17
19
|
end
|
18
20
|
|
data/bin/eyrestore
ADDED
@@ -0,0 +1,211 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
require 'yaml'
|
5
|
+
require 'tempfile'
|
6
|
+
require 'timeout'
|
7
|
+
|
8
|
+
def opt_parse(argv)
|
9
|
+
options = {}
|
10
|
+
@optparse = OptionParser.new do |opts|
|
11
|
+
opts.banner = "Usage: #{__FILE__} --env <environment name> --database <database_name> --action <action> [options]"
|
12
|
+
opts.separator ''
|
13
|
+
opts.separator "Wrapper for eybackup used for listing, downloading and restoring database backups. The main purpose for this tool is to"
|
14
|
+
opts.separator "simplify the process of accessing backups from one environment in a different environment (e.g. access Production backups"
|
15
|
+
opts.separator "from Staging). This can also be used for restoring backups within the current environment."
|
16
|
+
opts.separator ''
|
17
|
+
opts.on('-h', '--help', 'Displays this Help message') { puts opts; exit}
|
18
|
+
|
19
|
+
opts.separator ''
|
20
|
+
opts.separator 'Common Options:'
|
21
|
+
|
22
|
+
opts.on('-e', '--env environment_name', "Specifies the source environment name for the backups you will be working with.") { |env| options[:env] = env }
|
23
|
+
opts.on('-d', '--database database_name', "Name of the source database to list backups for.") { |dbname| options[:databases] = [dbname] }
|
24
|
+
actions=['list', 'restore', 'download']
|
25
|
+
opts.on('-a', '--action action_name', "The action you want to perform using the backup #{actions}.") do |action|
|
26
|
+
unless actions.include?(action)
|
27
|
+
puts "Invalid action: '#{action}' must be one of #{actions}\n\n"
|
28
|
+
puts opts
|
29
|
+
exit 1
|
30
|
+
end
|
31
|
+
options[:action] = action
|
32
|
+
end
|
33
|
+
opts.on('-i', '--index BACKUP_INDEX', 'Download/Restore the backup specified by index','BACKUP_INDEX uses the format #{index_number}:#{db_name}',
|
34
|
+
"The string 'last' will automatically reference the most recent available backup.") do |index|
|
35
|
+
options[:index] = index
|
36
|
+
end
|
37
|
+
opts.on('-f', '--force', 'Force mode, skips all confirmation prompts for use with automated restores.') { options[:force] = true }
|
38
|
+
|
39
|
+
opts.separator ''
|
40
|
+
opts.separator 'Additional Options:'
|
41
|
+
|
42
|
+
opts.on('-u', '--aws_secret_id key', 'AWS S3 Key') { |key| options[:aws_key_id] = key }
|
43
|
+
opts.on('-p', '--aws_secret_key secret', 'AWS S3 Secret Key') { |secret| options[:aws_secret_key] = secret }
|
44
|
+
opts.on('-b', '--backup_bucket bucket', 'AWS S3 bucket identifier') { |bucket| options[:backup_bucket] = bucket }
|
45
|
+
opts.on('-v', '--verbose', "Enable debug info for this wrapper.") { $verbose = true }
|
46
|
+
opts.on('-c', '--config', "Alternate config file for eyrestore (default ~/.eyrestore.confg.yml)") do |path|
|
47
|
+
unless File.exists?(path)
|
48
|
+
puts "Invalid config file path '#{path}', no file at that location."
|
49
|
+
exit 1
|
50
|
+
end
|
51
|
+
options[:def_config] = path
|
52
|
+
end
|
53
|
+
opts.separator ''
|
54
|
+
opts.separator 'Examples:'
|
55
|
+
opts.separator '* List backups from an environment named production for a database named todo'
|
56
|
+
opts.separator " sudo -i #{__FILE__} --env production --database todo --action list"
|
57
|
+
opts.separator ''
|
58
|
+
opts.separator '* Download the most recent backup from an environment named production for a database named todo'
|
59
|
+
opts.separator " sudo -i #{__FILE__} --env production --database todo --action download --index last"
|
60
|
+
opts.separator ''
|
61
|
+
opts.separator "* Download a backup from an environment named production for a database named todo, prompt with a list of available backups"
|
62
|
+
opts.separator " sudo -i #{__FILE__} --env production --database todo --action download"
|
63
|
+
opts.separator ''
|
64
|
+
opts.separator '* Restore the most recent backup from an environment named production for a database named todo'
|
65
|
+
opts.separator " sudo -i #{__FILE__} --env production --database todo --action restore --index last"
|
66
|
+
opts.separator ''
|
67
|
+
end
|
68
|
+
@optparse.parse!
|
69
|
+
options
|
70
|
+
end
|
71
|
+
|
72
|
+
def debug(msg)
|
73
|
+
puts '[' + Time.now.strftime("%D %T") + ']: DEBUG: ' + msg if $verbose
|
74
|
+
end
|
75
|
+
|
76
|
+
def find_engine
|
77
|
+
debug("Searching for database engine based on file '/etc/.*.backups.yml'")
|
78
|
+
main_config = Dir['/etc/.*.backups.yml'].first
|
79
|
+
abort "Failed to find eybackup config at '/etc/.*.backups.yml', this needs to run on a database instance." unless main_config
|
80
|
+
debug("Found file '#{main_config}'")
|
81
|
+
File.basename(main_config).split('.')[1]
|
82
|
+
end
|
83
|
+
|
84
|
+
def extra_validations(options)
|
85
|
+
mandatory = [:databases, :env, :action]
|
86
|
+
mandatory << :index if %w(download restore).include? options[:action]
|
87
|
+
missing = mandatory.select{ |param| options[param].nil? }
|
88
|
+
raise OptionParser::MissingArgument, missing.join(',') unless missing.empty?
|
89
|
+
end
|
90
|
+
|
91
|
+
def list_backups(engine, db, path)
|
92
|
+
command = "eybackup -e #{engine} -l #{db} -c #{path}"
|
93
|
+
debug("Listing backups with command: #{command}")
|
94
|
+
res = %x{#{command}}
|
95
|
+
last = res.chomp.split("\n").last
|
96
|
+
if last.match(/0 backup\(s\) found/)
|
97
|
+
abort "No Backups Found for that environment and database.\n Tips:\n - double check the source environment name is correct\n - you may have a legacy bucket (grep backup_bucket /etc/.*.backups.yml) in the source environment, pass to eyrestore with --backup_bucket"
|
98
|
+
end
|
99
|
+
res
|
100
|
+
end
|
101
|
+
|
102
|
+
def gets_timeout( prompt, secs )
|
103
|
+
print prompt + "[timeout=#{secs}secs]: "
|
104
|
+
Timeout::timeout( secs ) { gets.chomp }
|
105
|
+
rescue Timeout::Error
|
106
|
+
puts "*timeout"
|
107
|
+
'' # return nil if timeout
|
108
|
+
end
|
109
|
+
|
110
|
+
def backup_idx(listing, index)
|
111
|
+
if index == 'last'
|
112
|
+
last_backup_idx(listing)
|
113
|
+
else
|
114
|
+
index
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
def last_backup_idx(listing)
|
119
|
+
last = listing.chomp.split("\n").last
|
120
|
+
if last.split.first.match(/\d+:\w+/)
|
121
|
+
index = last.split.first
|
122
|
+
else
|
123
|
+
index = last.split[3]
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
options = opt_parse(ARGV)
|
128
|
+
options[:def_config] = '~/.eyrestore.config.yml' unless options[:def_config]
|
129
|
+
|
130
|
+
# auto-detect the type of database running in the given environment
|
131
|
+
engine = find_engine
|
132
|
+
|
133
|
+
# parse eybackup config file
|
134
|
+
config = YAML.load_file("/etc/.#{engine}.backups.yml")
|
135
|
+
this_env = config[:env]
|
136
|
+
# delete these just in case
|
137
|
+
config.delete(:env)
|
138
|
+
config.delete(:databases)
|
139
|
+
|
140
|
+
# override config from eyrestore config file
|
141
|
+
eyrestore = YAML.load_file(options[:def_config]) if File.exists?(options[:def_config])
|
142
|
+
config.merge!(eyrestore) unless eyrestore.nil?
|
143
|
+
|
144
|
+
# override config from command line options
|
145
|
+
config.merge!(options)
|
146
|
+
indexdb = config[:index].split(':')[1] if not config[:index].nil? and config[:index].include? ':'
|
147
|
+
config[:databases] = [indexdb] if config[:databases].nil? and not indexdb.nil? and indexdb.match(/^\w+$/)
|
148
|
+
begin
|
149
|
+
extra_validations(config)
|
150
|
+
rescue OptionParser::MissingArgument => e
|
151
|
+
puts e.message
|
152
|
+
puts @optparse
|
153
|
+
exit
|
154
|
+
end
|
155
|
+
debug("Config Options: '#{config}'")
|
156
|
+
|
157
|
+
# create a temporary configuration file from ~/.eyrestore.config.yml
|
158
|
+
temp_config = Tempfile.new("eyrestore")
|
159
|
+
temp_config.write(config.to_yaml)
|
160
|
+
temp_config.rewind
|
161
|
+
debug("Generated temporary config file: '#{temp_config.path}'")
|
162
|
+
|
163
|
+
db = config[:databases].first
|
164
|
+
temp = temp_config.path
|
165
|
+
if config[:action] == 'list' or ! config[:index]
|
166
|
+
puts list_backups(engine, db, temp)
|
167
|
+
end
|
168
|
+
|
169
|
+
# confirm environment overwrite if restoring
|
170
|
+
if config[:action] == 'restore' and ! config[:force]
|
171
|
+
response = gets_timeout("You are restoring the backup for '#{db}' from '#{config[:env]}' into '#{this_env}', THIS MAY BE DESTRUCTIVE; are you sure you want to proceed (Y/n) ? ", 30)
|
172
|
+
exit unless response.downcase == 'y'
|
173
|
+
end
|
174
|
+
|
175
|
+
|
176
|
+
if ['download','restore'].include?(config[:action])
|
177
|
+
config[:index] = gets_timeout("Enter the backup index to use for #{config[:action]} (e.g. 9:#{db}) ", 30) unless config[:index]
|
178
|
+
config[:index] = backup_idx(list_backups(engine, db, temp), config[:index])
|
179
|
+
# config[:index] = last_backup_idx(list_backups(engine, db, temp)) if config[:index] == 'last'
|
180
|
+
config[:index] = config[:index] + ":#{config[:databases].first}" if config[:index].match(/^\d+$/) and config[:databases] and config[:databases].size == 1
|
181
|
+
abort "Invalid backup index '#{config[:index]}'; proper format is <number>:<dbname>." unless config[:index].match(/^\d+:\w+$/)
|
182
|
+
|
183
|
+
action = config[:action] == 'restore' ? '-r' : '-d'
|
184
|
+
debug("Set action flag for eybackup to '#{action}' based on action of '#{config[:action]}'")
|
185
|
+
|
186
|
+
# test eybackup version for force option
|
187
|
+
ey_cloud_server_version = Gem::Version.new(%x{/usr/local/ey_resin/ruby/bin/gem list ey_cloud_server}.split[1].gsub('(','').gsub(')','').gsub(',',''))
|
188
|
+
force = ey_cloud_server_version >= Gem::Version.new('1.4.51') ? '--force' : ''
|
189
|
+
debug("eybackup version '#{ey_cloud_server_version}' so force is set to '#{force}'")
|
190
|
+
|
191
|
+
command="eybackup -e #{engine} -c #{temp} #{action} #{config[:index]} #{force}"
|
192
|
+
|
193
|
+
puts "Running '#{config[:action]}' on index '#{config[:index]}'."
|
194
|
+
debug("Running command: #{command}")
|
195
|
+
|
196
|
+
res=%x{#{command}}
|
197
|
+
puts res
|
198
|
+
|
199
|
+
last_line = res.split("\n").last
|
200
|
+
if last_line.nil?
|
201
|
+
# No-op, eybackup messaging is adequate.
|
202
|
+
elsif last_line.match(/^Filename/) and config[:action] == 'restore' and not last_line.match(/.gpz$/)
|
203
|
+
puts "Restore complete!"
|
204
|
+
else
|
205
|
+
puts "Download complete!"
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
# cleanup temporary config file
|
210
|
+
temp_config.close
|
211
|
+
temp_config.unlink
|
@@ -1,10 +1,12 @@
|
|
1
1
|
Given \
|
2
2
|
/^there is a mysql database \(([^\)]*)\)$/ do |db_key|
|
3
3
|
create_mysql_database(db_key)
|
4
|
+
setup_dna({:db_stack_name => "mysql5_5"})
|
4
5
|
end
|
5
6
|
|
6
7
|
Given \
|
7
8
|
/^the "([^\"]*)" mysql database is dropped$/ do |db_key|
|
8
9
|
db_name = created_mysql_dbs[db_key]
|
9
10
|
drop_mysql_database(db_name)
|
11
|
+
teardown_dna
|
10
12
|
end
|
data/features/support/env.rb
CHANGED
@@ -16,6 +16,10 @@ module EY
|
|
16
16
|
@bucket ||= @s3.directories.get(@bucket_name)
|
17
17
|
end
|
18
18
|
|
19
|
+
def file
|
20
|
+
bucket
|
21
|
+
end
|
22
|
+
|
19
23
|
def files
|
20
24
|
bucket.files
|
21
25
|
end
|
@@ -74,7 +78,13 @@ module EY
|
|
74
78
|
end
|
75
79
|
|
76
80
|
def put(filename, contents)
|
77
|
-
files.create(
|
81
|
+
files.create(
|
82
|
+
:key => filename,
|
83
|
+
:body => contents,
|
84
|
+
:public => false,
|
85
|
+
:multipart_chunk_size => 100*1024*1024, # 100MB
|
86
|
+
'x-amz-server-side-encryption' => 'AES256'
|
87
|
+
)
|
78
88
|
end
|
79
89
|
end
|
80
90
|
end
|
data/lib/ey-flex.rb
CHANGED
data/lib/ey_backup/backend.rb
CHANGED
@@ -8,7 +8,6 @@ module EY
|
|
8
8
|
|
9
9
|
def initialize(secret_id, secret_key, region, bucket_name)
|
10
10
|
@bucket_minder = EY::BucketMinder.new(secret_id, secret_key, bucket_name, region)
|
11
|
-
@s3 = Fog::Storage.new(:provider => 'AWS', :aws_access_key_id => secret_id, :aws_secret_access_key => secret_key, :region => region)
|
12
11
|
end
|
13
12
|
attr_reader :bucket_minder
|
14
13
|
|
@@ -17,13 +16,13 @@ module EY
|
|
17
16
|
begin
|
18
17
|
object_name = File.join("#{environment_name}.#{database_name}", "#{File.basename(filename)}")
|
19
18
|
info "Starting upload: #{filename}"
|
20
|
-
@
|
19
|
+
@bucket_minder.put(object_name, File.open(filename, 'r'))
|
21
20
|
info "Successful upload: #{filename}"
|
22
21
|
rescue => e
|
23
22
|
retries ||= 5
|
24
23
|
retries -= 1
|
25
24
|
# remove partial or failed uploads
|
26
|
-
@
|
25
|
+
@bucket_minder.remove_object(object_name)
|
27
26
|
raise e if retries == 0
|
28
27
|
warn "retrying upload of #{filename}. Got: #{e.inspect}"
|
29
28
|
retry
|
data/lib/ey_backup/backup_set.rb
CHANGED
@@ -31,26 +31,26 @@ module EY
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def self.list(databases)
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
puts
|
34
|
+
all_backups = []
|
35
|
+
databases.each do |db|
|
36
|
+
backups = db.backups
|
37
|
+
all_backups << backups
|
38
|
+
puts "Listing database backups for #{db.name}"
|
39
|
+
puts "#{backups.size} backup(s) found"
|
40
|
+
|
41
|
+
backups.each_with_index do |backup_set, i|
|
42
|
+
puts "#{i}:#{backup_set.database_name} #{backup_set.normalized_name}"
|
43
|
+
end
|
44
|
+
puts # just some extra whitespace
|
45
45
|
end
|
46
|
-
|
47
|
-
|
46
|
+
|
47
|
+
all_backups.flatten
|
48
48
|
end
|
49
49
|
|
50
50
|
def self.download(database, index)
|
51
|
-
fatal "You didn't specify a database name: e.g. 1:rails_production" if database.nil? || index
|
51
|
+
fatal "You didn't specify a database name: e.g. 1:rails_production" if database.nil? || index < 0
|
52
52
|
|
53
|
-
backup = database.backups[index
|
53
|
+
backup = database.backups[index]
|
54
54
|
|
55
55
|
fatal "No backup found for database #{database}: requested index: #{index}" unless backup
|
56
56
|
|
@@ -78,9 +78,9 @@ module EY
|
|
78
78
|
def download!
|
79
79
|
@remote_filenames = []
|
80
80
|
@keys.each do |key|
|
81
|
-
info "Downloading #{key} to #{@database.base_path}"
|
82
81
|
remote_filename = File.join(@database.base_path, normalize(key))
|
83
82
|
puts "Filename: #{remote_filename}"
|
83
|
+
info "Downloading #{key} to #{@database.base_path}"
|
84
84
|
File.open(remote_filename, 'wb') do |f|
|
85
85
|
@database.bucket_minder.stream(key) do |chunk, remaining_size, total_size|
|
86
86
|
f.write(chunk)
|
data/lib/ey_backup/cli.rb
CHANGED
@@ -9,18 +9,50 @@ module EY
|
|
9
9
|
|
10
10
|
config_path = options[:config] || "/etc/.#{options[:engine]}.backups.yml"
|
11
11
|
|
12
|
-
config_for(config_path).merge(options)
|
12
|
+
options = config_for(config_path).merge(options)
|
13
|
+
options = late_options.merge(options)
|
14
|
+
|
15
|
+
options[:tmp_dir] = '/mnt/tmp' if options[:tmp_dir].nil?
|
16
|
+
options = delayed_engine(File.join(options[:tmp_dir], '/chef/dna.json')).merge(options){|k, v1, v2| v2.nil? ? v1 : v2}
|
17
|
+
abort("Unable to determine database stack from dna; specify the engine with -e") if options[:engine].nil?
|
18
|
+
options
|
13
19
|
end
|
14
20
|
|
15
21
|
def default_options
|
16
22
|
{
|
17
23
|
:command => :new_backup,
|
18
24
|
:format => "gzip",
|
19
|
-
:engine => '
|
25
|
+
:engine => db_stack_name('/etc/chef/dna.json'),
|
20
26
|
:verbose => false,
|
21
27
|
}
|
22
28
|
end
|
23
29
|
|
30
|
+
def late_options
|
31
|
+
{
|
32
|
+
:log_coordinates => false,
|
33
|
+
:skip_analyze => false,
|
34
|
+
:allow_concurrent => false,
|
35
|
+
}
|
36
|
+
end
|
37
|
+
|
38
|
+
def delayed_engine(path)
|
39
|
+
{
|
40
|
+
:engine => db_stack_name(path),
|
41
|
+
}
|
42
|
+
end
|
43
|
+
|
44
|
+
def db_stack_name(file)
|
45
|
+
|
46
|
+
if File.exist?(file)
|
47
|
+
case %x{cat #{file} | grep db_stack_name}
|
48
|
+
when /mysql/
|
49
|
+
'mysql'
|
50
|
+
when /postgres/
|
51
|
+
'postgresql'
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
24
56
|
def verify_restore
|
25
57
|
res = ''
|
26
58
|
puts "This will overwrite your database, are you sure you would like to proceed (Y/n)?"
|
@@ -34,14 +66,16 @@ module EY
|
|
34
66
|
# Build a parser for the command line arguments
|
35
67
|
options = {}
|
36
68
|
|
37
|
-
|
69
|
+
optparse = OptionParser.new do |opts|
|
38
70
|
opts.version = EY::CloudServer::VERSION
|
39
71
|
|
40
72
|
opts.banner = "\nUsage: eybackup [-flag] [argument]"
|
41
|
-
opts.define_head " eybackup: manage
|
73
|
+
opts.define_head " eybackup: manage logical (mysqldump/pg_dump) style backups of your database.",
|
74
|
+
" When backing up multiple databases, each database is backed up separately."
|
42
75
|
opts.separator '*'*80
|
43
76
|
|
44
|
-
opts.on("-l", "--list-backup
|
77
|
+
opts.on("-l", "--list-backup DATABASE_NAME", "List backups for DATABASE_NAME; ",
|
78
|
+
" accepts 'all' to list all databases in the config (/etc/.*.backups.yml)") do |db|
|
45
79
|
if db == "all"
|
46
80
|
db = nil
|
47
81
|
end
|
@@ -52,7 +86,9 @@ module EY
|
|
52
86
|
opts.on("-e", "--engine DATABASE_ENGINE", "The database engine. ex: mysql, postgresql.") do |engine|
|
53
87
|
options[:engine] = engine
|
54
88
|
end
|
55
|
-
opts.on("-n", "--new-backup", "Create a new backup (default)"
|
89
|
+
opts.on("-n", "--new-backup [DATABASE_NAME]", "Create a new backup (default).",
|
90
|
+
" Backs up each database in '/etc/.*.backups.yml' if DATABASE_NAME not set.") do |db|
|
91
|
+
options[:db] = db.split(',') unless db.nil? or db == ''
|
56
92
|
options[:command] = :new_backup
|
57
93
|
end
|
58
94
|
|
@@ -89,9 +125,11 @@ module EY
|
|
89
125
|
' Run `eybackup -l #{db_name}` to get the index.',
|
90
126
|
' BACKUP_INDEX uses the format #{index_number}:#{db_name}') do |index_and_db|
|
91
127
|
options[:command] = :download
|
92
|
-
|
128
|
+
index, db = split_index(index_and_db)
|
129
|
+
index = index.to_i
|
93
130
|
options[:index] = index
|
94
131
|
options[:db] = db
|
132
|
+
raise OptionParser::InvalidArgument, "Index '#{index_and_db}' is not a valid format (hint: <number>:<dbname>)!" if index.nil? or not index.is_a? Numeric
|
95
133
|
end
|
96
134
|
|
97
135
|
opts.on("-r", "--restore BACKUP_INDEX", "Download and apply the backup specified by index.",
|
@@ -99,9 +137,11 @@ module EY
|
|
99
137
|
' Run `eybackup -l #{db_name}` to get the index.',
|
100
138
|
' BACKUP_INDEX uses the format #{index_number}:#{db_name}') do |index_and_db|
|
101
139
|
options[:command] = :restore
|
102
|
-
|
140
|
+
index, db = split_index(index_and_db)
|
141
|
+
index = index.to_i
|
103
142
|
options[:index] = index
|
104
143
|
options[:db] = db
|
144
|
+
raise OptionParser::InvalidArgument, "Index '#{index_and_db}' is not a valid format (hint: <number>:<dbname>)!" if index.nil? or not index.is_a? Numeric
|
105
145
|
end
|
106
146
|
|
107
147
|
options[:force] = false
|
@@ -109,16 +149,33 @@ module EY
|
|
109
149
|
" For use with automated restore operations (e.g. Staging).") do
|
110
150
|
options[:force] = true
|
111
151
|
end
|
152
|
+
|
153
|
+
opts.on("--log_coordinates", "Record MySQL binary log position so backup can be used with replication or Point in Time functions.") do
|
154
|
+
options[:log_coordinates] = true
|
155
|
+
end
|
156
|
+
|
157
|
+
opts.on("--allow_concurrent", "Allow eybackup process to run concurrently with other eybackup processes.") do
|
158
|
+
options[:allow_concurrent] = true
|
159
|
+
end
|
160
|
+
|
161
|
+
opts.on("--skip_analyze", "Skip automatic analyze during PostgreSQL Restore operations.") do
|
162
|
+
options[:skip_analyze] = true
|
163
|
+
end
|
112
164
|
|
113
165
|
end
|
114
166
|
|
115
|
-
|
167
|
+
begin
|
168
|
+
optparse.parse!(argv)
|
169
|
+
rescue => e
|
170
|
+
puts optparse
|
171
|
+
raise
|
172
|
+
end
|
116
173
|
|
117
174
|
options
|
118
175
|
end
|
119
176
|
|
120
177
|
def split_index(index)
|
121
|
-
index.split(':')
|
178
|
+
index.split(':')
|
122
179
|
end
|
123
180
|
|
124
181
|
def config_for(filename)
|
data/lib/ey_backup/dumper.rb
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
class String
|
2
|
+
def truncate(limit = 1)
|
3
|
+
self.match(%r{^(.{0,#{limit}})})[1]
|
4
|
+
end
|
5
|
+
end
|
6
|
+
|
1
7
|
module EY
|
2
8
|
module Backup
|
3
9
|
class Dumper < Base
|
@@ -26,15 +32,19 @@ module EY
|
|
26
32
|
|
27
33
|
alert_level=exceptions.empty? ? 'OKAY' : 'FAILURE'
|
28
34
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
35
|
+
# we don't provide alert here unless its a failure
|
36
|
+
unless alert_level == 'OKAY'
|
37
|
+
message.gsub!("\n", '\n')
|
38
|
+
message = Shellwords.escape(message).truncate(255)
|
39
|
+
full_txt= "Severity: #{alert_level}\n" \
|
40
|
+
+ "Time: #{Time.now.to_i}\n" \
|
41
|
+
+ "Type: process-dbbackup summary\n" \
|
42
|
+
+ "Plugin: exec\n"
|
43
|
+
full_txt = Shellwords.escape(full_txt)
|
44
|
+
full_txt += "raw_message:\\ \\'#{message}\\'"
|
45
|
+
alert_command = %Q(echo #{full_txt} | /engineyard/bin/ey-alert.rb 2>&1)
|
46
|
+
system(alert_command)
|
47
|
+
end
|
38
48
|
end
|
39
49
|
|
40
50
|
def initialize(database)
|
@@ -48,7 +58,7 @@ module EY
|
|
48
58
|
backup_set.split!(split_size)
|
49
59
|
backup_set.upload!
|
50
60
|
|
51
|
-
okay(
|
61
|
+
okay(@database.name, @database.backup_size)
|
52
62
|
backup_set.cleanup
|
53
63
|
backup_set.rm!
|
54
64
|
@database.backup_size
|
data/lib/ey_backup/engine.rb
CHANGED
@@ -3,7 +3,7 @@ module EY
|
|
3
3
|
class Engine < Base
|
4
4
|
include Spawner
|
5
5
|
|
6
|
-
attr_reader :username, :password, :host, :key_id, :force
|
6
|
+
attr_reader :username, :password, :host, :key_id, :force, :allow_concurrent, :skip_analyze, :log_coordinates
|
7
7
|
|
8
8
|
def self.label
|
9
9
|
@label
|
@@ -26,8 +26,21 @@ module EY
|
|
26
26
|
EY::Backup.logger.fatal("Unknown database engine: #{label}")
|
27
27
|
end
|
28
28
|
|
29
|
-
def initialize(username, password, host, key_id, force)
|
30
|
-
@username, @password, @host, @key_id, @force
|
29
|
+
def initialize(username, password, host, key_id, force, allow_concurrent, skip_analyze, log_coordinates)
|
30
|
+
@username, @password, @host, @key_id, @force, @allow_concurrent, @skip_analyze, @log_coordinates = username, password, host, key_id, force, allow_concurrent, skip_analyze, log_coordinates
|
31
|
+
end
|
32
|
+
|
33
|
+
def block_concurrent(db = nil)
|
34
|
+
if backup_running?
|
35
|
+
message = "Unable to backup #{db}: already a backup in progress. Use --allow_concurrent to enable concurrent backup runs."
|
36
|
+
error(message, db) unless File.exists?('/etc/engineyard/skip_concurrent_alerts') # this skips the dashboard alert
|
37
|
+
abort message
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def backup_running?
|
42
|
+
verbose %x{ps aux | grep [e]ybackup | grep rubygems}
|
43
|
+
%x{ps -ef | grep [e]ybackup | grep rubygems | wc -l}.to_i > 1
|
31
44
|
end
|
32
45
|
|
33
46
|
def gpg?
|