ey_cloud_server 1.4.54 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/bin/ey-snapshots +2 -1
- data/bin/eybackup +4 -2
- data/bin/eyrestore +211 -0
- data/features/step_definitions/mysql.rb +2 -0
- data/features/support/env.rb +3 -0
- data/lib/ey-flex/bucket_minder.rb +11 -1
- data/lib/ey-flex.rb +1 -1
- data/lib/ey_backup/backend.rb +2 -3
- data/lib/ey_backup/backup_set.rb +16 -16
- data/lib/ey_backup/cli.rb +67 -10
- data/lib/ey_backup/dumper.rb +20 -10
- data/lib/ey_backup/engine.rb +16 -3
- data/lib/ey_backup/engines/mysql_engine.rb +16 -5
- data/lib/ey_backup/engines/postgresql_engine.rb +37 -12
- data/lib/ey_backup/loader.rb +3 -1
- data/lib/ey_backup/logger.rb +42 -10
- data/lib/ey_backup/processors/splitter.rb +1 -1
- data/lib/ey_backup/spawner.rb +2 -5
- data/lib/ey_backup.rb +13 -2
- data/lib/ey_cloud_server/version.rb +1 -1
- data/spec/config.yml.ci +11 -0
- data/spec/ey_backup/backup_spec.rb +2 -0
- data/spec/ey_backup/cli_spec.rb +2 -0
- data/spec/ey_backup/mysql_backups_spec.rb +26 -0
- data/spec/ey_backup/postgres_backups_spec.rb +12 -0
- data/spec/helpers.rb +40 -14
- data/spec/spec_helper.rb +3 -0
- metadata +33 -37
- data/spec/config.yml +0 -11
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
ZmFkNjBkYjY1OTNhZjQwNjMxNjUxMmNmOTRmOWQyZDMxMDJlNTUzYQ==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
YTA0NTBmMTAxYzk5ZmE5YjQ0MzViYjE4NWE2YjRjZGM5YWYxZDNiNQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
ZmRlZTQ1YThjMWU0N2RkMzVlNmFkNGY3YThmMDJhMzRjZTE4MzU4ZGJjOTAx
|
10
|
+
NzM5MDc4OGZmNTMxNzM1OGI1ZTk4NDgxZjY5MTIwYmZmMTI1NmNjNjc0NmFl
|
11
|
+
OTI3OWFmMjdiZGE0N2QwZmEyNjljYzQ0NTg0MjBmZWI3NTBiOWU=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
MTdlZDkwYmQ1ZmNiYTJmY2U0YWMzMmQ2YjZmNzA5NWEzYTVlNTA4ODM1ZGZj
|
14
|
+
ZTBkN2EwZmVlODczYTAxMjYwNzA3OTk3ODcxODlkOTQwZjYwZDNhNjE4Mjk2
|
15
|
+
YThjMTZiNjBmMmY3NTE5Yjg0NjI2YjRkOGQwMDg0YmQ4OTY3ZGM=
|
data/bin/ey-snapshots
CHANGED
data/bin/eybackup
CHANGED
@@ -12,7 +12,9 @@ require File.dirname(__FILE__) + '/../lib/ey-flex'
|
|
12
12
|
begin
|
13
13
|
EY::Backup.run(ARGV)
|
14
14
|
rescue => e
|
15
|
-
|
16
|
-
|
15
|
+
puts e.message
|
16
|
+
puts "******* Trace *******"
|
17
|
+
puts e.inspect
|
18
|
+
EY.notify_backup_error(e) unless STDOUT.isatty
|
17
19
|
end
|
18
20
|
|
data/bin/eyrestore
ADDED
@@ -0,0 +1,211 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
require 'yaml'
|
5
|
+
require 'tempfile'
|
6
|
+
require 'timeout'
|
7
|
+
|
8
|
+
def opt_parse(argv)
|
9
|
+
options = {}
|
10
|
+
@optparse = OptionParser.new do |opts|
|
11
|
+
opts.banner = "Usage: #{__FILE__} --env <environment name> --database <database_name> --action <action> [options]"
|
12
|
+
opts.separator ''
|
13
|
+
opts.separator "Wrapper for eybackup used for listing, downloading and restoring database backups. The main purpose for this tool is to"
|
14
|
+
opts.separator "simplify the process of accessing backups from one environment in a different environment (e.g. access Production backups"
|
15
|
+
opts.separator "from Staging). This can also be used for restoring backups within the current environment."
|
16
|
+
opts.separator ''
|
17
|
+
opts.on('-h', '--help', 'Displays this Help message') { puts opts; exit}
|
18
|
+
|
19
|
+
opts.separator ''
|
20
|
+
opts.separator 'Common Options:'
|
21
|
+
|
22
|
+
opts.on('-e', '--env environment_name', "Specifies the source environment name for the backups you will be working with.") { |env| options[:env] = env }
|
23
|
+
opts.on('-d', '--database database_name', "Name of the source database to list backups for.") { |dbname| options[:databases] = [dbname] }
|
24
|
+
actions=['list', 'restore', 'download']
|
25
|
+
opts.on('-a', '--action action_name', "The action you want to perform using the backup #{actions}.") do |action|
|
26
|
+
unless actions.include?(action)
|
27
|
+
puts "Invalid action: '#{action}' must be one of #{actions}\n\n"
|
28
|
+
puts opts
|
29
|
+
exit 1
|
30
|
+
end
|
31
|
+
options[:action] = action
|
32
|
+
end
|
33
|
+
opts.on('-i', '--index BACKUP_INDEX', 'Download/Restore the backup specified by index','BACKUP_INDEX uses the format #{index_number}:#{db_name}',
|
34
|
+
"The string 'last' will automatically reference the most recent available backup.") do |index|
|
35
|
+
options[:index] = index
|
36
|
+
end
|
37
|
+
opts.on('-f', '--force', 'Force mode, skips all confirmation prompts for use with automated restores.') { options[:force] = true }
|
38
|
+
|
39
|
+
opts.separator ''
|
40
|
+
opts.separator 'Additional Options:'
|
41
|
+
|
42
|
+
opts.on('-u', '--aws_secret_id key', 'AWS S3 Key') { |key| options[:aws_key_id] = key }
|
43
|
+
opts.on('-p', '--aws_secret_key secret', 'AWS S3 Secret Key') { |secret| options[:aws_secret_key] = secret }
|
44
|
+
opts.on('-b', '--backup_bucket bucket', 'AWS S3 bucket identifier') { |bucket| options[:backup_bucket] = bucket }
|
45
|
+
opts.on('-v', '--verbose', "Enable debug info for this wrapper.") { $verbose = true }
|
46
|
+
opts.on('-c', '--config', "Alternate config file for eyrestore (default ~/.eyrestore.confg.yml)") do |path|
|
47
|
+
unless File.exists?(path)
|
48
|
+
puts "Invalid config file path '#{path}', no file at that location."
|
49
|
+
exit 1
|
50
|
+
end
|
51
|
+
options[:def_config] = path
|
52
|
+
end
|
53
|
+
opts.separator ''
|
54
|
+
opts.separator 'Examples:'
|
55
|
+
opts.separator '* List backups from an environment named production for a database named todo'
|
56
|
+
opts.separator " sudo -i #{__FILE__} --env production --database todo --action list"
|
57
|
+
opts.separator ''
|
58
|
+
opts.separator '* Download the most recent backup from an environment named production for a database named todo'
|
59
|
+
opts.separator " sudo -i #{__FILE__} --env production --database todo --action download --index last"
|
60
|
+
opts.separator ''
|
61
|
+
opts.separator "* Download a backup from an environment named production for a database named todo, prompt with a list of available backups"
|
62
|
+
opts.separator " sudo -i #{__FILE__} --env production --database todo --action download"
|
63
|
+
opts.separator ''
|
64
|
+
opts.separator '* Restore the most recent backup from an environment named production for a database named todo'
|
65
|
+
opts.separator " sudo -i #{__FILE__} --env production --database todo --action restore --index last"
|
66
|
+
opts.separator ''
|
67
|
+
end
|
68
|
+
@optparse.parse!
|
69
|
+
options
|
70
|
+
end
|
71
|
+
|
72
|
+
def debug(msg)
|
73
|
+
puts '[' + Time.now.strftime("%D %T") + ']: DEBUG: ' + msg if $verbose
|
74
|
+
end
|
75
|
+
|
76
|
+
def find_engine
|
77
|
+
debug("Searching for database engine based on file '/etc/.*.backups.yml'")
|
78
|
+
main_config = Dir['/etc/.*.backups.yml'].first
|
79
|
+
abort "Failed to find eybackup config at '/etc/.*.backups.yml', this needs to run on a database instance." unless main_config
|
80
|
+
debug("Found file '#{main_config}'")
|
81
|
+
File.basename(main_config).split('.')[1]
|
82
|
+
end
|
83
|
+
|
84
|
+
def extra_validations(options)
|
85
|
+
mandatory = [:databases, :env, :action]
|
86
|
+
mandatory << :index if %w(download restore).include? options[:action]
|
87
|
+
missing = mandatory.select{ |param| options[param].nil? }
|
88
|
+
raise OptionParser::MissingArgument, missing.join(',') unless missing.empty?
|
89
|
+
end
|
90
|
+
|
91
|
+
def list_backups(engine, db, path)
|
92
|
+
command = "eybackup -e #{engine} -l #{db} -c #{path}"
|
93
|
+
debug("Listing backups with command: #{command}")
|
94
|
+
res = %x{#{command}}
|
95
|
+
last = res.chomp.split("\n").last
|
96
|
+
if last.match(/0 backup\(s\) found/)
|
97
|
+
abort "No Backups Found for that environment and database.\n Tips:\n - double check the source environment name is correct\n - you may have a legacy bucket (grep backup_bucket /etc/.*.backups.yml) in the source environment, pass to eyrestore with --backup_bucket"
|
98
|
+
end
|
99
|
+
res
|
100
|
+
end
|
101
|
+
|
102
|
+
def gets_timeout( prompt, secs )
|
103
|
+
print prompt + "[timeout=#{secs}secs]: "
|
104
|
+
Timeout::timeout( secs ) { gets.chomp }
|
105
|
+
rescue Timeout::Error
|
106
|
+
puts "*timeout"
|
107
|
+
'' # return nil if timeout
|
108
|
+
end
|
109
|
+
|
110
|
+
def backup_idx(listing, index)
|
111
|
+
if index == 'last'
|
112
|
+
last_backup_idx(listing)
|
113
|
+
else
|
114
|
+
index
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
def last_backup_idx(listing)
|
119
|
+
last = listing.chomp.split("\n").last
|
120
|
+
if last.split.first.match(/\d+:\w+/)
|
121
|
+
index = last.split.first
|
122
|
+
else
|
123
|
+
index = last.split[3]
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
options = opt_parse(ARGV)
|
128
|
+
options[:def_config] = '~/.eyrestore.config.yml' unless options[:def_config]
|
129
|
+
|
130
|
+
# auto-detect the type of database running in the given environment
|
131
|
+
engine = find_engine
|
132
|
+
|
133
|
+
# parse eybackup config file
|
134
|
+
config = YAML.load_file("/etc/.#{engine}.backups.yml")
|
135
|
+
this_env = config[:env]
|
136
|
+
# delete these just in case
|
137
|
+
config.delete(:env)
|
138
|
+
config.delete(:databases)
|
139
|
+
|
140
|
+
# override config from eyrestore config file
|
141
|
+
eyrestore = YAML.load_file(options[:def_config]) if File.exists?(options[:def_config])
|
142
|
+
config.merge!(eyrestore) unless eyrestore.nil?
|
143
|
+
|
144
|
+
# override config from command line options
|
145
|
+
config.merge!(options)
|
146
|
+
indexdb = config[:index].split(':')[1] if not config[:index].nil? and config[:index].include? ':'
|
147
|
+
config[:databases] = [indexdb] if config[:databases].nil? and not indexdb.nil? and indexdb.match(/^\w+$/)
|
148
|
+
begin
|
149
|
+
extra_validations(config)
|
150
|
+
rescue OptionParser::MissingArgument => e
|
151
|
+
puts e.message
|
152
|
+
puts @optparse
|
153
|
+
exit
|
154
|
+
end
|
155
|
+
debug("Config Options: '#{config}'")
|
156
|
+
|
157
|
+
# create a temporary configuration file from ~/.eyrestore.config.yml
|
158
|
+
temp_config = Tempfile.new("eyrestore")
|
159
|
+
temp_config.write(config.to_yaml)
|
160
|
+
temp_config.rewind
|
161
|
+
debug("Generated temporary config file: '#{temp_config.path}'")
|
162
|
+
|
163
|
+
db = config[:databases].first
|
164
|
+
temp = temp_config.path
|
165
|
+
if config[:action] == 'list' or ! config[:index]
|
166
|
+
puts list_backups(engine, db, temp)
|
167
|
+
end
|
168
|
+
|
169
|
+
# confirm environment overwrite if restoring
|
170
|
+
if config[:action] == 'restore' and ! config[:force]
|
171
|
+
response = gets_timeout("You are restoring the backup for '#{db}' from '#{config[:env]}' into '#{this_env}', THIS MAY BE DESTRUCTIVE; are you sure you want to proceed (Y/n) ? ", 30)
|
172
|
+
exit unless response.downcase == 'y'
|
173
|
+
end
|
174
|
+
|
175
|
+
|
176
|
+
if ['download','restore'].include?(config[:action])
|
177
|
+
config[:index] = gets_timeout("Enter the backup index to use for #{config[:action]} (e.g. 9:#{db}) ", 30) unless config[:index]
|
178
|
+
config[:index] = backup_idx(list_backups(engine, db, temp), config[:index])
|
179
|
+
# config[:index] = last_backup_idx(list_backups(engine, db, temp)) if config[:index] == 'last'
|
180
|
+
config[:index] = config[:index] + ":#{config[:databases].first}" if config[:index].match(/^\d+$/) and config[:databases] and config[:databases].size == 1
|
181
|
+
abort "Invalid backup index '#{config[:index]}'; proper format is <number>:<dbname>." unless config[:index].match(/^\d+:\w+$/)
|
182
|
+
|
183
|
+
action = config[:action] == 'restore' ? '-r' : '-d'
|
184
|
+
debug("Set action flag for eybackup to '#{action}' based on action of '#{config[:action]}'")
|
185
|
+
|
186
|
+
# test eybackup version for force option
|
187
|
+
ey_cloud_server_version = Gem::Version.new(%x{/usr/local/ey_resin/ruby/bin/gem list ey_cloud_server}.split[1].gsub('(','').gsub(')','').gsub(',',''))
|
188
|
+
force = ey_cloud_server_version >= Gem::Version.new('1.4.51') ? '--force' : ''
|
189
|
+
debug("eybackup version '#{ey_cloud_server_version}' so force is set to '#{force}'")
|
190
|
+
|
191
|
+
command="eybackup -e #{engine} -c #{temp} #{action} #{config[:index]} #{force}"
|
192
|
+
|
193
|
+
puts "Running '#{config[:action]}' on index '#{config[:index]}'."
|
194
|
+
debug("Running command: #{command}")
|
195
|
+
|
196
|
+
res=%x{#{command}}
|
197
|
+
puts res
|
198
|
+
|
199
|
+
last_line = res.split("\n").last
|
200
|
+
if last_line.nil?
|
201
|
+
# No-op, eybackup messaging is adequate.
|
202
|
+
elsif last_line.match(/^Filename/) and config[:action] == 'restore' and not last_line.match(/.gpz$/)
|
203
|
+
puts "Restore complete!"
|
204
|
+
else
|
205
|
+
puts "Download complete!"
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
# cleanup temporary config file
|
210
|
+
temp_config.close
|
211
|
+
temp_config.unlink
|
@@ -1,10 +1,12 @@
|
|
1
1
|
Given \
|
2
2
|
/^there is a mysql database \(([^\)]*)\)$/ do |db_key|
|
3
3
|
create_mysql_database(db_key)
|
4
|
+
setup_dna({:db_stack_name => "mysql5_5"})
|
4
5
|
end
|
5
6
|
|
6
7
|
Given \
|
7
8
|
/^the "([^\"]*)" mysql database is dropped$/ do |db_key|
|
8
9
|
db_name = created_mysql_dbs[db_key]
|
9
10
|
drop_mysql_database(db_name)
|
11
|
+
teardown_dna
|
10
12
|
end
|
data/features/support/env.rb
CHANGED
@@ -16,6 +16,10 @@ module EY
|
|
16
16
|
@bucket ||= @s3.directories.get(@bucket_name)
|
17
17
|
end
|
18
18
|
|
19
|
+
def file
|
20
|
+
bucket
|
21
|
+
end
|
22
|
+
|
19
23
|
def files
|
20
24
|
bucket.files
|
21
25
|
end
|
@@ -74,7 +78,13 @@ module EY
|
|
74
78
|
end
|
75
79
|
|
76
80
|
def put(filename, contents)
|
77
|
-
files.create(
|
81
|
+
files.create(
|
82
|
+
:key => filename,
|
83
|
+
:body => contents,
|
84
|
+
:public => false,
|
85
|
+
:multipart_chunk_size => 100*1024*1024, # 100MB
|
86
|
+
'x-amz-server-side-encryption' => 'AES256'
|
87
|
+
)
|
78
88
|
end
|
79
89
|
end
|
80
90
|
end
|
data/lib/ey-flex.rb
CHANGED
data/lib/ey_backup/backend.rb
CHANGED
@@ -8,7 +8,6 @@ module EY
|
|
8
8
|
|
9
9
|
def initialize(secret_id, secret_key, region, bucket_name)
|
10
10
|
@bucket_minder = EY::BucketMinder.new(secret_id, secret_key, bucket_name, region)
|
11
|
-
@s3 = Fog::Storage.new(:provider => 'AWS', :aws_access_key_id => secret_id, :aws_secret_access_key => secret_key, :region => region)
|
12
11
|
end
|
13
12
|
attr_reader :bucket_minder
|
14
13
|
|
@@ -17,13 +16,13 @@ module EY
|
|
17
16
|
begin
|
18
17
|
object_name = File.join("#{environment_name}.#{database_name}", "#{File.basename(filename)}")
|
19
18
|
info "Starting upload: #{filename}"
|
20
|
-
@
|
19
|
+
@bucket_minder.put(object_name, File.open(filename, 'r'))
|
21
20
|
info "Successful upload: #{filename}"
|
22
21
|
rescue => e
|
23
22
|
retries ||= 5
|
24
23
|
retries -= 1
|
25
24
|
# remove partial or failed uploads
|
26
|
-
@
|
25
|
+
@bucket_minder.remove_object(object_name)
|
27
26
|
raise e if retries == 0
|
28
27
|
warn "retrying upload of #{filename}. Got: #{e.inspect}"
|
29
28
|
retry
|
data/lib/ey_backup/backup_set.rb
CHANGED
@@ -31,26 +31,26 @@ module EY
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def self.list(databases)
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
puts
|
34
|
+
all_backups = []
|
35
|
+
databases.each do |db|
|
36
|
+
backups = db.backups
|
37
|
+
all_backups << backups
|
38
|
+
puts "Listing database backups for #{db.name}"
|
39
|
+
puts "#{backups.size} backup(s) found"
|
40
|
+
|
41
|
+
backups.each_with_index do |backup_set, i|
|
42
|
+
puts "#{i}:#{backup_set.database_name} #{backup_set.normalized_name}"
|
43
|
+
end
|
44
|
+
puts # just some extra whitespace
|
45
45
|
end
|
46
|
-
|
47
|
-
|
46
|
+
|
47
|
+
all_backups.flatten
|
48
48
|
end
|
49
49
|
|
50
50
|
def self.download(database, index)
|
51
|
-
fatal "You didn't specify a database name: e.g. 1:rails_production" if database.nil? || index
|
51
|
+
fatal "You didn't specify a database name: e.g. 1:rails_production" if database.nil? || index < 0
|
52
52
|
|
53
|
-
backup = database.backups[index
|
53
|
+
backup = database.backups[index]
|
54
54
|
|
55
55
|
fatal "No backup found for database #{database}: requested index: #{index}" unless backup
|
56
56
|
|
@@ -78,9 +78,9 @@ module EY
|
|
78
78
|
def download!
|
79
79
|
@remote_filenames = []
|
80
80
|
@keys.each do |key|
|
81
|
-
info "Downloading #{key} to #{@database.base_path}"
|
82
81
|
remote_filename = File.join(@database.base_path, normalize(key))
|
83
82
|
puts "Filename: #{remote_filename}"
|
83
|
+
info "Downloading #{key} to #{@database.base_path}"
|
84
84
|
File.open(remote_filename, 'wb') do |f|
|
85
85
|
@database.bucket_minder.stream(key) do |chunk, remaining_size, total_size|
|
86
86
|
f.write(chunk)
|
data/lib/ey_backup/cli.rb
CHANGED
@@ -9,18 +9,50 @@ module EY
|
|
9
9
|
|
10
10
|
config_path = options[:config] || "/etc/.#{options[:engine]}.backups.yml"
|
11
11
|
|
12
|
-
config_for(config_path).merge(options)
|
12
|
+
options = config_for(config_path).merge(options)
|
13
|
+
options = late_options.merge(options)
|
14
|
+
|
15
|
+
options[:tmp_dir] = '/mnt/tmp' if options[:tmp_dir].nil?
|
16
|
+
options = delayed_engine(File.join(options[:tmp_dir], '/chef/dna.json')).merge(options){|k, v1, v2| v2.nil? ? v1 : v2}
|
17
|
+
abort("Unable to determine database stack from dna; specify the engine with -e") if options[:engine].nil?
|
18
|
+
options
|
13
19
|
end
|
14
20
|
|
15
21
|
def default_options
|
16
22
|
{
|
17
23
|
:command => :new_backup,
|
18
24
|
:format => "gzip",
|
19
|
-
:engine => '
|
25
|
+
:engine => db_stack_name('/etc/chef/dna.json'),
|
20
26
|
:verbose => false,
|
21
27
|
}
|
22
28
|
end
|
23
29
|
|
30
|
+
def late_options
|
31
|
+
{
|
32
|
+
:log_coordinates => false,
|
33
|
+
:skip_analyze => false,
|
34
|
+
:allow_concurrent => false,
|
35
|
+
}
|
36
|
+
end
|
37
|
+
|
38
|
+
def delayed_engine(path)
|
39
|
+
{
|
40
|
+
:engine => db_stack_name(path),
|
41
|
+
}
|
42
|
+
end
|
43
|
+
|
44
|
+
def db_stack_name(file)
|
45
|
+
|
46
|
+
if File.exist?(file)
|
47
|
+
case %x{cat #{file} | grep db_stack_name}
|
48
|
+
when /mysql/
|
49
|
+
'mysql'
|
50
|
+
when /postgres/
|
51
|
+
'postgresql'
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
24
56
|
def verify_restore
|
25
57
|
res = ''
|
26
58
|
puts "This will overwrite your database, are you sure you would like to proceed (Y/n)?"
|
@@ -34,14 +66,16 @@ module EY
|
|
34
66
|
# Build a parser for the command line arguments
|
35
67
|
options = {}
|
36
68
|
|
37
|
-
|
69
|
+
optparse = OptionParser.new do |opts|
|
38
70
|
opts.version = EY::CloudServer::VERSION
|
39
71
|
|
40
72
|
opts.banner = "\nUsage: eybackup [-flag] [argument]"
|
41
|
-
opts.define_head " eybackup: manage
|
73
|
+
opts.define_head " eybackup: manage logical (mysqldump/pg_dump) style backups of your database.",
|
74
|
+
" When backing up multiple databases, each database is backed up separately."
|
42
75
|
opts.separator '*'*80
|
43
76
|
|
44
|
-
opts.on("-l", "--list-backup
|
77
|
+
opts.on("-l", "--list-backup DATABASE_NAME", "List backups for DATABASE_NAME; ",
|
78
|
+
" accepts 'all' to list all databases in the config (/etc/.*.backups.yml)") do |db|
|
45
79
|
if db == "all"
|
46
80
|
db = nil
|
47
81
|
end
|
@@ -52,7 +86,9 @@ module EY
|
|
52
86
|
opts.on("-e", "--engine DATABASE_ENGINE", "The database engine. ex: mysql, postgresql.") do |engine|
|
53
87
|
options[:engine] = engine
|
54
88
|
end
|
55
|
-
opts.on("-n", "--new-backup", "Create a new backup (default)"
|
89
|
+
opts.on("-n", "--new-backup [DATABASE_NAME]", "Create a new backup (default).",
|
90
|
+
" Backs up each database in '/etc/.*.backups.yml' if DATABASE_NAME not set.") do |db|
|
91
|
+
options[:db] = db.split(',') unless db.nil? or db == ''
|
56
92
|
options[:command] = :new_backup
|
57
93
|
end
|
58
94
|
|
@@ -89,9 +125,11 @@ module EY
|
|
89
125
|
' Run `eybackup -l #{db_name}` to get the index.',
|
90
126
|
' BACKUP_INDEX uses the format #{index_number}:#{db_name}') do |index_and_db|
|
91
127
|
options[:command] = :download
|
92
|
-
|
128
|
+
index, db = split_index(index_and_db)
|
129
|
+
index = index.to_i
|
93
130
|
options[:index] = index
|
94
131
|
options[:db] = db
|
132
|
+
raise OptionParser::InvalidArgument, "Index '#{index_and_db}' is not a valid format (hint: <number>:<dbname>)!" if index.nil? or not index.is_a? Numeric
|
95
133
|
end
|
96
134
|
|
97
135
|
opts.on("-r", "--restore BACKUP_INDEX", "Download and apply the backup specified by index.",
|
@@ -99,9 +137,11 @@ module EY
|
|
99
137
|
' Run `eybackup -l #{db_name}` to get the index.',
|
100
138
|
' BACKUP_INDEX uses the format #{index_number}:#{db_name}') do |index_and_db|
|
101
139
|
options[:command] = :restore
|
102
|
-
|
140
|
+
index, db = split_index(index_and_db)
|
141
|
+
index = index.to_i
|
103
142
|
options[:index] = index
|
104
143
|
options[:db] = db
|
144
|
+
raise OptionParser::InvalidArgument, "Index '#{index_and_db}' is not a valid format (hint: <number>:<dbname>)!" if index.nil? or not index.is_a? Numeric
|
105
145
|
end
|
106
146
|
|
107
147
|
options[:force] = false
|
@@ -109,16 +149,33 @@ module EY
|
|
109
149
|
" For use with automated restore operations (e.g. Staging).") do
|
110
150
|
options[:force] = true
|
111
151
|
end
|
152
|
+
|
153
|
+
opts.on("--log_coordinates", "Record MySQL binary log position so backup can be used with replication or Point in Time functions.") do
|
154
|
+
options[:log_coordinates] = true
|
155
|
+
end
|
156
|
+
|
157
|
+
opts.on("--allow_concurrent", "Allow eybackup process to run concurrently with other eybackup processes.") do
|
158
|
+
options[:allow_concurrent] = true
|
159
|
+
end
|
160
|
+
|
161
|
+
opts.on("--skip_analyze", "Skip automatic analyze during PostgreSQL Restore operations.") do
|
162
|
+
options[:skip_analyze] = true
|
163
|
+
end
|
112
164
|
|
113
165
|
end
|
114
166
|
|
115
|
-
|
167
|
+
begin
|
168
|
+
optparse.parse!(argv)
|
169
|
+
rescue => e
|
170
|
+
puts optparse
|
171
|
+
raise
|
172
|
+
end
|
116
173
|
|
117
174
|
options
|
118
175
|
end
|
119
176
|
|
120
177
|
def split_index(index)
|
121
|
-
index.split(':')
|
178
|
+
index.split(':')
|
122
179
|
end
|
123
180
|
|
124
181
|
def config_for(filename)
|
data/lib/ey_backup/dumper.rb
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
class String
|
2
|
+
def truncate(limit = 1)
|
3
|
+
self.match(%r{^(.{0,#{limit}})})[1]
|
4
|
+
end
|
5
|
+
end
|
6
|
+
|
1
7
|
module EY
|
2
8
|
module Backup
|
3
9
|
class Dumper < Base
|
@@ -26,15 +32,19 @@ module EY
|
|
26
32
|
|
27
33
|
alert_level=exceptions.empty? ? 'OKAY' : 'FAILURE'
|
28
34
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
35
|
+
# we don't provide alert here unless its a failure
|
36
|
+
unless alert_level == 'OKAY'
|
37
|
+
message.gsub!("\n", '\n')
|
38
|
+
message = Shellwords.escape(message).truncate(255)
|
39
|
+
full_txt= "Severity: #{alert_level}\n" \
|
40
|
+
+ "Time: #{Time.now.to_i}\n" \
|
41
|
+
+ "Type: process-dbbackup summary\n" \
|
42
|
+
+ "Plugin: exec\n"
|
43
|
+
full_txt = Shellwords.escape(full_txt)
|
44
|
+
full_txt += "raw_message:\\ \\'#{message}\\'"
|
45
|
+
alert_command = %Q(echo #{full_txt} | /engineyard/bin/ey-alert.rb 2>&1)
|
46
|
+
system(alert_command)
|
47
|
+
end
|
38
48
|
end
|
39
49
|
|
40
50
|
def initialize(database)
|
@@ -48,7 +58,7 @@ module EY
|
|
48
58
|
backup_set.split!(split_size)
|
49
59
|
backup_set.upload!
|
50
60
|
|
51
|
-
okay(
|
61
|
+
okay(@database.name, @database.backup_size)
|
52
62
|
backup_set.cleanup
|
53
63
|
backup_set.rm!
|
54
64
|
@database.backup_size
|
data/lib/ey_backup/engine.rb
CHANGED
@@ -3,7 +3,7 @@ module EY
|
|
3
3
|
class Engine < Base
|
4
4
|
include Spawner
|
5
5
|
|
6
|
-
attr_reader :username, :password, :host, :key_id, :force
|
6
|
+
attr_reader :username, :password, :host, :key_id, :force, :allow_concurrent, :skip_analyze, :log_coordinates
|
7
7
|
|
8
8
|
def self.label
|
9
9
|
@label
|
@@ -26,8 +26,21 @@ module EY
|
|
26
26
|
EY::Backup.logger.fatal("Unknown database engine: #{label}")
|
27
27
|
end
|
28
28
|
|
29
|
-
def initialize(username, password, host, key_id, force)
|
30
|
-
@username, @password, @host, @key_id, @force
|
29
|
+
def initialize(username, password, host, key_id, force, allow_concurrent, skip_analyze, log_coordinates)
|
30
|
+
@username, @password, @host, @key_id, @force, @allow_concurrent, @skip_analyze, @log_coordinates = username, password, host, key_id, force, allow_concurrent, skip_analyze, log_coordinates
|
31
|
+
end
|
32
|
+
|
33
|
+
def block_concurrent(db = nil)
|
34
|
+
if backup_running?
|
35
|
+
message = "Unable to backup #{db}: already a backup in progress. Use --allow_concurrent to enable concurrent backup runs."
|
36
|
+
error(message, db) unless File.exists?('/etc/engineyard/skip_concurrent_alerts') # this skips the dashboard alert
|
37
|
+
abort message
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def backup_running?
|
42
|
+
verbose %x{ps aux | grep [e]ybackup | grep rubygems}
|
43
|
+
%x{ps -ef | grep [e]ybackup | grep rubygems | wc -l}.to_i > 1
|
31
44
|
end
|
32
45
|
|
33
46
|
def gpg?
|