ey_cloud_server 1.4.54 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/bin/ey-snapshots +2 -1
- data/bin/eybackup +4 -2
- data/bin/eyrestore +211 -0
- data/features/step_definitions/mysql.rb +2 -0
- data/features/support/env.rb +3 -0
- data/lib/ey-flex/bucket_minder.rb +11 -1
- data/lib/ey-flex.rb +1 -1
- data/lib/ey_backup/backend.rb +2 -3
- data/lib/ey_backup/backup_set.rb +16 -16
- data/lib/ey_backup/cli.rb +67 -10
- data/lib/ey_backup/dumper.rb +20 -10
- data/lib/ey_backup/engine.rb +16 -3
- data/lib/ey_backup/engines/mysql_engine.rb +16 -5
- data/lib/ey_backup/engines/postgresql_engine.rb +37 -12
- data/lib/ey_backup/loader.rb +3 -1
- data/lib/ey_backup/logger.rb +42 -10
- data/lib/ey_backup/processors/splitter.rb +1 -1
- data/lib/ey_backup/spawner.rb +2 -5
- data/lib/ey_backup.rb +13 -2
- data/lib/ey_cloud_server/version.rb +1 -1
- data/spec/config.yml.ci +11 -0
- data/spec/ey_backup/backup_spec.rb +2 -0
- data/spec/ey_backup/cli_spec.rb +2 -0
- data/spec/ey_backup/mysql_backups_spec.rb +26 -0
- data/spec/ey_backup/postgres_backups_spec.rb +12 -0
- data/spec/helpers.rb +40 -14
- data/spec/spec_helper.rb +3 -0
- metadata +33 -37
- data/spec/config.yml +0 -11
@@ -6,7 +6,7 @@ module EY
|
|
6
6
|
def dump(database_name, basename)
|
7
7
|
file = basename + '.sql'
|
8
8
|
|
9
|
-
command = "mysqldump #{username_option} #{host_option} #{single_transaction_option(database_name)} #{routines_option} #{master_data_option} #{databases_option(database_name)}
|
9
|
+
command = "( #{server_id} mysqldump #{username_option} #{host_option} #{single_transaction_option(database_name)} #{routines_option} #{master_data_option} #{databases_option(database_name)} ) "
|
10
10
|
|
11
11
|
if gpg?
|
12
12
|
command << " | " << GPGEncryptor.command_for(key_id)
|
@@ -17,6 +17,8 @@ module EY
|
|
17
17
|
end
|
18
18
|
|
19
19
|
command << " > #{file}"
|
20
|
+
|
21
|
+
block_concurrent(database_name) unless allow_concurrent
|
20
22
|
|
21
23
|
run(command, database_name)
|
22
24
|
|
@@ -27,14 +29,16 @@ module EY
|
|
27
29
|
command = "cat #{file}"
|
28
30
|
|
29
31
|
if file =~ /.gpz$/ # GPG?
|
30
|
-
abort "
|
32
|
+
abort "\nCannot restore a GPG backup directly; decrypt the file (#{file}) using your key and then load using the mysql client.
|
33
|
+
To decrypt a backup: https://support.cloud.engineyard.com/hc/en-us/articles/205413948-Use-PGP-Encrypted-Database-Backups-with-Engine-Yard-Cloud#restore
|
34
|
+
Once decrypted, restore with: `gunzip -f < <filename> | mysql #{username_option} #{host_option} #{database_name}`\n\n"
|
31
35
|
else
|
32
36
|
command << " | " << GZipper.gunzip
|
33
37
|
end
|
34
38
|
|
35
39
|
cycle_database(database_name)
|
36
40
|
|
37
|
-
command << " | mysql #{username_option} #{host_option} #{database_name}
|
41
|
+
command << " | mysql #{username_option} #{host_option} #{database_name} "
|
38
42
|
|
39
43
|
run(command, database_name)
|
40
44
|
end
|
@@ -52,12 +56,19 @@ module EY
|
|
52
56
|
"--routines"
|
53
57
|
end
|
54
58
|
|
59
|
+
def server_id
|
60
|
+
if log_bin_on? and log_coordinates
|
61
|
+
stdout = %x{mysql #{username_option} #{host_option} -BN -e"select @@global.server_id"}
|
62
|
+
"echo '-- Server_id: #{stdout.strip}' && "
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
55
66
|
def master_data_option
|
56
|
-
"--master-data=2" if log_bin_on?
|
67
|
+
"--master-data=2" if log_bin_on? and log_coordinates
|
57
68
|
end
|
58
69
|
|
59
70
|
def databases_option(db)
|
60
|
-
"
|
71
|
+
"#{db}"
|
61
72
|
end
|
62
73
|
|
63
74
|
def host_option
|
@@ -6,7 +6,7 @@ module EY
|
|
6
6
|
def dump(database_name, basename)
|
7
7
|
file = basename + '.dump'
|
8
8
|
|
9
|
-
command = "PGPASSWORD='#{password}' pg_dump -h #{host} --create --format=c -
|
9
|
+
command = "PGPASSWORD='#{password}' pg_dump -h #{host} --create --format=c -U#{username} #{database_name} "
|
10
10
|
|
11
11
|
if gpg?
|
12
12
|
command << " | " << GPGEncryptor.command_for(key_id)
|
@@ -15,6 +15,7 @@ module EY
|
|
15
15
|
|
16
16
|
command << " > #{file}"
|
17
17
|
|
18
|
+
block_concurrent(database_name) unless allow_concurrent
|
18
19
|
run(command, database_name)
|
19
20
|
|
20
21
|
file
|
@@ -22,25 +23,39 @@ module EY
|
|
22
23
|
|
23
24
|
def load(database_name, file)
|
24
25
|
if file =~ /.gpz$/ # GPG?
|
25
|
-
abort "
|
26
|
+
abort "\nCannot restore a GPG backup directly; decrypt the file (#{file}) using your key and then load with pg_restore.
|
27
|
+
To decrypt a backup: https://support.cloud.engineyard.com/hc/en-us/articles/205413948-Use-PGP-Encrypted-Database-Backups-with-Engine-Yard-Cloud#restore
|
28
|
+
Once decrypted, restore with: `pg_restore -h #{host} --format=c --clean -U#{username} -d #{database_name} <filename>`\n\n"
|
26
29
|
end
|
27
30
|
|
28
31
|
cycle_database(database_name)
|
32
|
+
|
33
|
+
# Exclude Extension Comments
|
34
|
+
toc_file = "#{file}.toc"
|
35
|
+
table_of_contents(file, toc_file)
|
29
36
|
|
30
37
|
command = "cat #{file}"
|
31
38
|
|
32
|
-
command << " | PGPASSWORD='#{password}' pg_restore -h #{host} --format=c -
|
39
|
+
command << " | PGPASSWORD='#{password}' pg_restore -L #{toc_file} -h #{host} --format=c -U#{username} -d #{database_name}"
|
33
40
|
|
34
41
|
run(command, database_name)
|
42
|
+
|
43
|
+
system("rm #{toc_file}") if File.exists?(toc_file)
|
44
|
+
|
45
|
+
# Analyze database unless disabled
|
46
|
+
unless skip_analyze
|
47
|
+
verbose "Analyzing database '#{database_name}', use --skip-analyze to skip this step."
|
48
|
+
%x{PGPASSWORD='#{password}' vacuumdb -h #{host} -U#{username} -d #{database_name} --analyze-only}
|
49
|
+
end
|
35
50
|
end
|
36
51
|
|
37
52
|
def check_connections(database_name)
|
38
|
-
active_connections = %x{PGPASSWORD='#{password}' psql -U
|
53
|
+
active_connections = %x{PGPASSWORD='#{password}' psql -U#{username} -h #{host} -t postgres -c "select count(*) from pg_stat_activity where datname='#{database_name}';"}
|
39
54
|
|
40
55
|
if active_connections.to_i > 0
|
41
56
|
res = ''
|
42
57
|
unless force
|
43
|
-
puts "There are currently #{
|
58
|
+
puts "There are currently #{active_connections} connections on database: '#{database_name}'; can I kill these to continue (Y/n):"
|
44
59
|
Timeout::timeout(30){
|
45
60
|
res = gets.strip
|
46
61
|
}
|
@@ -55,13 +70,13 @@ module EY
|
|
55
70
|
end
|
56
71
|
|
57
72
|
def cancel_connections(database_name)
|
58
|
-
%x{psql -U
|
73
|
+
%x{psql -U#{username} -h #{host} postgres -c"SELECT pg_terminate_backend(pg_stat_activity.pid)
|
59
74
|
FROM pg_stat_activity
|
60
75
|
WHERE pg_stat_activity.datname = '#{database_name}';"}
|
61
76
|
end
|
62
77
|
|
63
78
|
def drop_database(database_name)
|
64
|
-
command = "PGPASSWORD='#{password}' dropdb -h #{host} -
|
79
|
+
command = "PGPASSWORD='#{password}' dropdb -h #{host} -U#{username} #{database_name}"
|
65
80
|
verbose "Dropping Database with: #{command}"
|
66
81
|
%x{#{command}}
|
67
82
|
end
|
@@ -73,14 +88,23 @@ module EY
|
|
73
88
|
end
|
74
89
|
|
75
90
|
def check_if_replica
|
76
|
-
|
91
|
+
command = "PGPASSWORD='#{password}' psql -U#{username} -h #{host} -t -c 'select pg_is_in_recovery()' postgres| head -n 1"
|
92
|
+
verbose "Checking for replica state with: #{command}"
|
93
|
+
stdout = %x{#{command}}
|
77
94
|
unless stdout.chomp =~ /^\W*f$/
|
78
95
|
EY::Backup.logger.fatal(%Q{ERROR: Target host: '#{host}' is currently a replica in recovery mode; restore operations need to be processed against the master.})
|
79
96
|
end
|
80
97
|
end
|
81
98
|
|
99
|
+
def table_of_contents(file, toc_file)
|
100
|
+
command = %Q{pg_restore -l #{file} | sed -e 's/^\\\(.* COMMENT - EXTENSION .*\\\)/;\\1/g' \
|
101
|
+
-e 's/^\\\(.* rdsadmin$\\\)/;\\1/g' > #{toc_file}}
|
102
|
+
verbose "Creating table of contents: #{command}"
|
103
|
+
%x{#{command}}
|
104
|
+
end
|
105
|
+
|
82
106
|
def create_command(database_name)
|
83
|
-
|
107
|
+
command="PGPASSWORD='#{password}' psql -U#{username} -h #{host} -t postgres -c \"SELECT 'CREATE DATABASE ' || datname ||
|
84
108
|
' WITH OWNER ' || pg_user.usename ||
|
85
109
|
CASE (select pg_encoding_to_char(encoding) from pg_database where datname='template1')
|
86
110
|
WHEN pg_encoding_to_char(encoding)
|
@@ -94,8 +118,9 @@ module EY
|
|
94
118
|
FROM pg_database
|
95
119
|
INNER JOIN pg_user
|
96
120
|
ON pg_user.usesysid = pg_database.datdba
|
97
|
-
WHERE datname = '#{database_name}'"
|
98
|
-
}
|
121
|
+
WHERE datname = '#{database_name}'\""
|
122
|
+
verbose "Getting create info: #{command}"
|
123
|
+
%x{#{command}}
|
99
124
|
end
|
100
125
|
|
101
126
|
def cycle_database(database_name)
|
@@ -105,7 +130,7 @@ module EY
|
|
105
130
|
else
|
106
131
|
check_connections(database_name)
|
107
132
|
drop_database(database_name)
|
108
|
-
%x{PGPASSWORD='#{password}' psql -U
|
133
|
+
%x{PGPASSWORD='#{password}' psql -U#{username} -h #{host} -t postgres -c "#{create_cmd}"}
|
109
134
|
end
|
110
135
|
end
|
111
136
|
|
data/lib/ey_backup/loader.rb
CHANGED
data/lib/ey_backup/logger.rb
CHANGED
@@ -1,8 +1,15 @@
|
|
1
|
+
class String
|
2
|
+
def truncate(limit = 1)
|
3
|
+
self.match(%r{^(.{0,#{limit}})})[1]
|
4
|
+
end
|
5
|
+
end
|
6
|
+
|
1
7
|
module EY
|
2
8
|
module Backup
|
3
9
|
class Logger
|
4
10
|
extend Forwardable
|
5
11
|
require 'shellwords'
|
12
|
+
require 'fileutils'
|
6
13
|
|
7
14
|
attr_accessor :stdout, :stderr
|
8
15
|
|
@@ -22,21 +29,22 @@ module EY
|
|
22
29
|
|
23
30
|
def push_dashboard_alert(message, alert_level, db = nil)
|
24
31
|
message.gsub!("\n", '\n')
|
32
|
+
message = Shellwords.escape(message).truncate(255)
|
25
33
|
type="process-dbbackup"
|
26
34
|
type = type + " #{db}" unless db.nil?
|
27
35
|
full_txt= "Severity: #{alert_level}\n" \
|
28
36
|
+ "Time: #{Time.now.to_i}\n" \
|
29
37
|
+ "Type: #{type}\n" \
|
30
|
-
+ "Plugin: exec\n"
|
31
|
-
+ "raw_message: '#{message}'"
|
38
|
+
+ "Plugin: exec\n"
|
32
39
|
full_txt = Shellwords.escape(full_txt)
|
40
|
+
full_txt += "raw_message:\\ \\'#{message}\\'"
|
33
41
|
alert_command = %Q(echo #{full_txt} | /engineyard/bin/ey-alert.rb 2>&1)
|
34
42
|
verbose "Sending dashboard alert"
|
35
43
|
system(alert_command)
|
36
44
|
end
|
37
45
|
|
38
46
|
def info(msg)
|
39
|
-
puts "#{Time.now} #{msg}"
|
47
|
+
stdout.puts "#{Time.now} #{msg}"
|
40
48
|
end
|
41
49
|
|
42
50
|
def verbose(msg)
|
@@ -46,24 +54,48 @@ module EY
|
|
46
54
|
def set_verbose()
|
47
55
|
@verbose = true
|
48
56
|
end
|
57
|
+
|
58
|
+
def set_log_path=(path)
|
59
|
+
@status_path = path
|
60
|
+
FileUtils.mkdir_p(@status_path)
|
61
|
+
end
|
49
62
|
|
50
63
|
def say(msg, newline = true)
|
51
64
|
newline ? info(msg) : stdout.print(msg)
|
52
65
|
end
|
53
66
|
|
54
|
-
def okay(
|
55
|
-
|
56
|
-
|
67
|
+
def okay(db, size)
|
68
|
+
filepath = File.join(@status_path, "#{db}.sizes")
|
69
|
+
%x{LOG=$(tail -n 100 #{filepath}); echo "$LOG" > #{filepath}} if File.exists?(filepath)
|
70
|
+
%x{echo "#{Time.now()} #{size}" >> #{filepath}}
|
71
|
+
msg = "Backup successful for '#{db}' after previous failure."
|
72
|
+
push_dashboard_alert(msg, 'OKAY', db) if clear_alert?(db)
|
57
73
|
end
|
58
74
|
|
59
75
|
def warn(msg, db = nil)
|
60
|
-
|
76
|
+
stdout.puts("#{Time.now} WARNING: #{msg}")
|
77
|
+
set_alert(msg, db) unless db.nil?
|
61
78
|
push_dashboard_alert(msg, "WARNING", db)
|
62
79
|
end
|
63
80
|
|
64
81
|
def error(msg, db = nil)
|
65
|
-
|
66
|
-
|
82
|
+
stdout.puts("#{Time.now} ERROR: #{msg}")
|
83
|
+
set_alert(msg, db) unless db.nil?
|
84
|
+
push_dashboard_alert("#{msg} Details at /var/log/eybackup.log.", "FAILURE", db)
|
85
|
+
end
|
86
|
+
|
87
|
+
def set_alert(msg, db)
|
88
|
+
fullPath = File.join(@status_path, "#{db}.alert")
|
89
|
+
File.open(fullPath, 'a') { |file| file.puts "#{Time.now}: #{msg}"}
|
90
|
+
end
|
91
|
+
|
92
|
+
def clear_alert?(db)
|
93
|
+
begin
|
94
|
+
File.delete(File.join(@status_path, "#{db}.alert"))
|
95
|
+
true
|
96
|
+
rescue Errno::ENOENT
|
97
|
+
false
|
98
|
+
end
|
67
99
|
end
|
68
100
|
|
69
101
|
def exception(type, msg)
|
@@ -71,7 +103,7 @@ module EY
|
|
71
103
|
end
|
72
104
|
|
73
105
|
def debug(msg)
|
74
|
-
|
106
|
+
stdout.puts("#{Time.now} DEBUG: #{msg}")
|
75
107
|
end
|
76
108
|
|
77
109
|
end
|
data/lib/ey_backup/spawner.rb
CHANGED
@@ -42,14 +42,11 @@ module EY
|
|
42
42
|
pid, stdin, stdout, stderr = Open4.popen4("bash -o pipefail -c #{escaped_command}")
|
43
43
|
pid, status = Process::waitpid2(pid)
|
44
44
|
|
45
|
-
verbose "stdout: #{stdout.read}"
|
46
|
-
verbose "stderr: #{stderr.read}"
|
47
45
|
verbose "status: #{status}"
|
48
46
|
|
49
47
|
if ! status.success?
|
50
|
-
dumperr = File.exists?("/tmp/eybackup.#{pid}.dumperr") ? File.read("/tmp/eybackup.#{pid}.dumperr") : status
|
51
|
-
err_msg = "
|
52
|
-
verbose "#{db} backup failed: #{err_msg}"
|
48
|
+
dumperr = File.exists?("/tmp/eybackup.#{pid}.dumperr") ? File.read("/tmp/eybackup.#{pid}.dumperr") : "#{status}: #{stderr.read.chomp}: #{stdout.read.chomp}"
|
49
|
+
err_msg = "#{db} backup failed! The error returned was: #{dumperr}"
|
53
50
|
error(err_msg, db)
|
54
51
|
end
|
55
52
|
|
data/lib/ey_backup.rb
CHANGED
@@ -44,6 +44,7 @@ module EY
|
|
44
44
|
class << self
|
45
45
|
attr_accessor :logger
|
46
46
|
attr_accessor :tmp_dir
|
47
|
+
attr_accessor :log_dir
|
47
48
|
end
|
48
49
|
|
49
50
|
def self.run(argv = ARGV)
|
@@ -96,11 +97,12 @@ module EY
|
|
96
97
|
if @options[:db].nil? || @options[:db].empty?
|
97
98
|
@options[:databases]
|
98
99
|
else
|
99
|
-
[@options[:db]]
|
100
|
+
[@options[:db]].flatten
|
100
101
|
end
|
101
102
|
end
|
102
103
|
|
103
104
|
def setup
|
105
|
+
setup_log_dir
|
104
106
|
setup_logger
|
105
107
|
setup_tmp_dir
|
106
108
|
setup_backend
|
@@ -117,6 +119,7 @@ module EY
|
|
117
119
|
EY::Backup.logger.set_verbose
|
118
120
|
end
|
119
121
|
end
|
122
|
+
EY::Backup.logger.set_log_path=EY::Backup.log_dir
|
120
123
|
end
|
121
124
|
|
122
125
|
def setup_tmp_dir
|
@@ -126,6 +129,14 @@ module EY
|
|
126
129
|
EY::Backup.tmp_dir = "/mnt/tmp"
|
127
130
|
end
|
128
131
|
end
|
132
|
+
|
133
|
+
def setup_log_dir
|
134
|
+
if @options[:log_dir]
|
135
|
+
EY::Backup.log_dir = @options[:log_dir]
|
136
|
+
else
|
137
|
+
EY::Backup.log_dir = "/var/log/engineyard/eybackup"
|
138
|
+
end
|
139
|
+
end
|
129
140
|
|
130
141
|
def setup_backend
|
131
142
|
@backend = Backend.new(@options[:aws_secret_id], @options[:aws_secret_key], @options[:region], @options[:backup_bucket])
|
@@ -136,7 +147,7 @@ module EY
|
|
136
147
|
if ! @options.key?(:dbhost) or @options[:dbhost] == nil or @options[:dbhost] == ""
|
137
148
|
@options[:dbhost] = 'localhost'
|
138
149
|
end
|
139
|
-
@engine = engine_class.new(@options[:dbuser], @options[:dbpass], @options[:dbhost], @options[:key_id], @options[:force])
|
150
|
+
@engine = engine_class.new(@options[:dbuser], @options[:dbpass], @options[:dbhost], @options[:key_id], @options[:force], @options[:allow_concurrent], @options[:skip_analyze], @options[:log_coordinates])
|
140
151
|
end
|
141
152
|
|
142
153
|
def dispatch
|
data/spec/config.yml.ci
ADDED
@@ -3,10 +3,12 @@ require File.dirname(__FILE__) + '/spec_helper'
|
|
3
3
|
describe EY::Backup do
|
4
4
|
before(:each) do
|
5
5
|
@db_name = create_mysql_database('first')
|
6
|
+
setup_dna({:db_stack_name => "mysql5_5"})
|
6
7
|
end
|
7
8
|
|
8
9
|
after(:each) do
|
9
10
|
drop_mysql_database(@db_name)
|
11
|
+
teardown_dna
|
10
12
|
end
|
11
13
|
|
12
14
|
describe "#list" do
|
data/spec/ey_backup/cli_spec.rb
CHANGED
@@ -3,10 +3,12 @@ require File.dirname(__FILE__) + '/spec_helper'
|
|
3
3
|
describe "MySQL Backups" do
|
4
4
|
before(:each) do
|
5
5
|
@db_name = create_mysql_database('first')
|
6
|
+
setup_dna({:db_stack_name => "mysql5_5"})
|
6
7
|
end
|
7
8
|
|
8
9
|
after(:each) do
|
9
10
|
drop_mysql_database(@db_name)
|
11
|
+
teardown_dna
|
10
12
|
end
|
11
13
|
|
12
14
|
describe "--quiet" do
|
@@ -3,11 +3,13 @@ require File.dirname(__FILE__) + '/spec_helper'
|
|
3
3
|
describe "MySQL Backups" do
|
4
4
|
before(:each) do
|
5
5
|
@db_name = create_mysql_database('first')
|
6
|
+
setup_dna({:db_stack_name => "mysql5_5"})
|
6
7
|
mk_tmp
|
7
8
|
end
|
8
9
|
|
9
10
|
after(:each) do
|
10
11
|
drop_mysql_database(@db_name)
|
12
|
+
teardown_dna
|
11
13
|
end
|
12
14
|
|
13
15
|
it "makes a backup" do
|
@@ -125,17 +127,37 @@ describe "MySQL Backups" do
|
|
125
127
|
run_sql("SELECT * FROM `bar`;", @db_name).should be_true
|
126
128
|
FileUtils.rm(file)
|
127
129
|
end
|
130
|
+
|
131
|
+
it 'detects a backup failure due to an invalid view definition and reports it to stdout' do
|
132
|
+
|
133
|
+
run_sql("CREATE TABLE `foo` (`id` int(11) NOT NULL auto_increment, PRIMARY KEY(`id`));", @db_name).should be_truthy
|
134
|
+
run_sql("CREATE TABLE `bar` (`id` int(11) NOT NULL auto_increment, PRIMARY KEY(`id`));", @db_name).should be_truthy
|
135
|
+
run_sql("CREATE view `vw_foobar` as select `foo`.`id` as `foo_id`, `bar`.`id` as `bar_id` from `foo` inner join `bar` on `foo`.`id` = `bar`.`id`;", @db_name).should be_truthy
|
136
|
+
run_sql("DROP TABLE `bar`;", @db_name).should be_truthy
|
137
|
+
|
138
|
+
reset_logger
|
139
|
+
|
140
|
+
EY::Backup.run(["-c", backup_config_file])
|
141
|
+
stdout.should match(/mysqldump: Couldn't execute 'SHOW FIELDS FROM `vw_foobar`'/)
|
142
|
+
|
143
|
+
files = Dir["#{EY::Backup.tmp_dir}/*#{@db_name}*"]
|
144
|
+
|
145
|
+
files.size.should == 1
|
146
|
+
FileUtils.rm(files.first)
|
147
|
+
end
|
128
148
|
end
|
129
149
|
|
130
150
|
describe "MySQL Backups" do
|
131
151
|
before(:each) do
|
132
152
|
@dbs = [create_mysql_database('first'), create_mysql_database('second')]
|
153
|
+
setup_dna({:db_stack_name => "mysql5_5"})
|
133
154
|
end
|
134
155
|
|
135
156
|
after(:each) do
|
136
157
|
@dbs.each do |db|
|
137
158
|
drop_mysql_database(db)
|
138
159
|
end
|
160
|
+
teardown_dna
|
139
161
|
end
|
140
162
|
|
141
163
|
it "makes a backup" do
|
@@ -154,10 +176,12 @@ describe "a multi-region backup" do
|
|
154
176
|
describe "in japan!" do
|
155
177
|
before(:each) do
|
156
178
|
@db_name = create_mysql_database('speaking_japanese', 'ap-northeast-1')
|
179
|
+
setup_dna({:db_stack_name => "mysql5_5"})
|
157
180
|
end
|
158
181
|
|
159
182
|
after(:each) do
|
160
183
|
drop_mysql_database(@db_name)
|
184
|
+
teardown_dna
|
161
185
|
end
|
162
186
|
|
163
187
|
it "makes a split backup" do
|
@@ -193,10 +217,12 @@ describe "a multi-region backup" do
|
|
193
217
|
describe "in europe!" do
|
194
218
|
before(:each) do
|
195
219
|
@db_name = create_mysql_database('speaking_esperanto', 'eu-west-1')
|
220
|
+
setup_dna({:db_stack_name => "mysql5_5"})
|
196
221
|
end
|
197
222
|
|
198
223
|
after(:each) do
|
199
224
|
drop_mysql_database(@db_name)
|
225
|
+
teardown_dna
|
200
226
|
end
|
201
227
|
|
202
228
|
it "makes a split backup" do
|
@@ -3,10 +3,12 @@ require File.dirname(__FILE__) + '/spec_helper'
|
|
3
3
|
describe "Postgres Backups" do
|
4
4
|
before(:each) do
|
5
5
|
@db_name = create_postgresql_database('first')
|
6
|
+
setup_dna({:db_stack_name => "postgres9_5"})
|
6
7
|
end
|
7
8
|
|
8
9
|
after(:each) do
|
9
10
|
drop_postgresql_database(@db_name)
|
11
|
+
teardown_dna
|
10
12
|
end
|
11
13
|
|
12
14
|
it "makes a custom format backup" do
|
@@ -18,6 +20,16 @@ describe "Postgres Backups" do
|
|
18
20
|
|
19
21
|
stdout.should match(/0:#{@db_name}.*\.dump$/)
|
20
22
|
end
|
23
|
+
|
24
|
+
it "makes a custom backup and detects that the db is Postgres" do
|
25
|
+
EY::Backup.run(["-c", backup_config_file])
|
26
|
+
|
27
|
+
reset_logger
|
28
|
+
|
29
|
+
EY::Backup.run(["-c", backup_config_file, "-l", @db_name, "-e", "postgresql"])
|
30
|
+
|
31
|
+
stdout.should match(/0:#{@db_name}.*\.dump$/)
|
32
|
+
end
|
21
33
|
|
22
34
|
it "makes a split backup" do
|
23
35
|
EY::Backup.run(["-c", backup_config_file, "-e", "postgresql", "-s", "100"])
|
data/spec/helpers.rb
CHANGED
@@ -19,18 +19,28 @@ module Helpers
|
|
19
19
|
@mock_environment_name => {:id => 1}
|
20
20
|
}))
|
21
21
|
|
22
|
-
|
23
|
-
|
22
|
+
create_or_clean_dir(tmp_dir)
|
23
|
+
create_or_clean_dir(log_dir)
|
24
|
+
|
25
|
+
# FileUtils.mkdir_p(tmp_dir)
|
26
|
+
# FileUtils.mkdir_p(log_dir)
|
27
|
+
# Dir.glob("#{tmp_dir}/*").each do |f|
|
28
|
+
# FileUtils.rm(f)
|
29
|
+
# end
|
30
|
+
stub_configs
|
31
|
+
end
|
32
|
+
|
33
|
+
def create_or_clean_dir(dir)
|
34
|
+
FileUtils.mkdir_p(dir)
|
35
|
+
Dir.glob("#{dir}/*").each do |f|
|
24
36
|
FileUtils.rm(f)
|
25
37
|
end
|
26
|
-
|
27
|
-
stub_configs
|
28
38
|
end
|
29
39
|
|
30
40
|
def stub_configs
|
31
41
|
#print "in stub_configs: #{YAML::dump(@database_config)}, #{YAML::dump(spec_config)}\n"
|
32
42
|
config = @database_config || spec_config
|
33
|
-
config = config.merge({ :tmp_dir => tmp_dir })
|
43
|
+
config = config.merge({ :tmp_dir => tmp_dir, :log_dir => log_dir })
|
34
44
|
# print "in stub_configs2: #{YAML::dump(config)}\n"
|
35
45
|
File.open(backup_config_file, "w") do |f|
|
36
46
|
f.puts YAML.dump(config )
|
@@ -38,6 +48,9 @@ module Helpers
|
|
38
48
|
end
|
39
49
|
|
40
50
|
def import_gpg
|
51
|
+
# GPG v2.1+ dropped support for the --secret-keyring option (https://lists.gnupg.org/pipermail/gnupg-devel/2014-December/029296.html)
|
52
|
+
# This means that we need to import private keys too
|
53
|
+
system("gpg --import #{PRIVATE_KEY_PATH}") || raise("Could not import public key")
|
41
54
|
system("gpg --import #{PUBLIC_KEY_PATH}") || raise("Could not import public key")
|
42
55
|
end
|
43
56
|
|
@@ -55,7 +68,7 @@ module Helpers
|
|
55
68
|
return File.read(PUBLIC_KEY_PATH), File.read(PRIVATE_KEY_PATH)
|
56
69
|
end
|
57
70
|
|
58
|
-
def run_after
|
71
|
+
def run_after(skip_file_remove = false)
|
59
72
|
FileUtils.rm_f(backup_config_file)
|
60
73
|
filenames = Dir.glob("#{tmp_dir}/*")
|
61
74
|
if filenames.any?
|
@@ -63,16 +76,17 @@ module Helpers
|
|
63
76
|
end
|
64
77
|
end
|
65
78
|
|
66
|
-
def teardown_dna(
|
67
|
-
FileUtils.rm_f("#{tmp_dir}/dna.json")
|
79
|
+
def teardown_dna()
|
80
|
+
FileUtils.rm_f("#{tmp_dir}/chef/dna.json")
|
81
|
+
FileUtils.rm_r("#{tmp_dir}/chef")
|
68
82
|
end
|
69
83
|
|
70
84
|
def setup_dna(data)
|
71
|
-
FileUtils.mkdir_p("#{tmp_dir}")
|
72
|
-
if File.exist?("#{tmp_dir}/dna.json")
|
73
|
-
FileUtils.rm("#{tmp_dir}/dna.json")
|
85
|
+
FileUtils.mkdir_p("#{tmp_dir}/chef")
|
86
|
+
if File.exist?("#{tmp_dir}/chef/dna.json")
|
87
|
+
FileUtils.rm("#{tmp_dir}/chef/dna.json")
|
74
88
|
end
|
75
|
-
File.open("#{tmp_dir}/dna.json", "w") do |f|
|
89
|
+
File.open("#{tmp_dir}/chef/dna.json", "w") do |f|
|
76
90
|
f.puts data.to_json
|
77
91
|
end
|
78
92
|
end
|
@@ -80,7 +94,7 @@ module Helpers
|
|
80
94
|
def backup_config_file
|
81
95
|
"#{tmp_dir}/spec_backups.yml"
|
82
96
|
end
|
83
|
-
|
97
|
+
|
84
98
|
def mk_tmp
|
85
99
|
FileUtils.mkdir_p("/tmp")
|
86
100
|
end
|
@@ -102,7 +116,7 @@ module Helpers
|
|
102
116
|
write_database_config('mysql', mysql_user, mysql_password, mysql_host, created_mysql_dbs.values, region)
|
103
117
|
db_name
|
104
118
|
end
|
105
|
-
|
119
|
+
|
106
120
|
def drop_mysql_database(db_name)
|
107
121
|
command = %Q{mysql -u#{mysql_user} -h#{mysql_host} #{mysql_password_option} -e "drop database if exists #{db_name};"}
|
108
122
|
puts "*** MySQL Drop Command: #{command}"
|
@@ -184,6 +198,14 @@ module Helpers
|
|
184
198
|
end
|
185
199
|
end
|
186
200
|
|
201
|
+
def log_dir
|
202
|
+
if spec_config['log_dir'].nil?
|
203
|
+
File.dirname(__FILE__) + "/log/"
|
204
|
+
else
|
205
|
+
spec_config['log_dir'] || File.dirname("./log/")
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
187
209
|
def mysql_user
|
188
210
|
spec_config['mysql_user'] || "root"
|
189
211
|
end
|
@@ -281,5 +303,9 @@ module Helpers
|
|
281
303
|
EY::Backup.logger.stdout.string
|
282
304
|
end
|
283
305
|
|
306
|
+
def stderr
|
307
|
+
EY::Backup.logger.stderr.string
|
308
|
+
end
|
309
|
+
|
284
310
|
load_spec_config
|
285
311
|
end
|