new_backup 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +6 -0
- data/.rvmrc +48 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +82 -0
- data/LICENSE.txt +7 -0
- data/README.md +56 -0
- data/README.rdoc +53 -0
- data/Rakefile +70 -0
- data/bin/new_backup +47 -0
- data/features/new_backup.feature +14 -0
- data/features/step_definitions/new_backup_steps.rb +1 -0
- data/features/support/env.rb +16 -0
- data/lib/new_backup.rb +10 -0
- data/lib/new_backup/datadog.rb +92 -0
- data/lib/new_backup/main.rb +193 -0
- data/lib/new_backup/myrds.rb +140 -0
- data/lib/new_backup/mys3.rb +123 -0
- data/lib/new_backup/mysqlcmds.rb +95 -0
- data/lib/new_backup/version.rb +17 -0
- data/new_backup.gemspec +27 -0
- data/sample-config.yml +20 -0
- data/spec/datadog_spec.rb +31 -0
- data/spec/main_spec.rb +77 -0
- data/spec/mockbucket.rb +83 -0
- data/spec/myrds_spec.rb +131 -0
- data/spec/mys3_spec.rb +123 -0
- data/spec/mysqlcmds_spec.rb +119 -0
- data/spec/spec_helper.rb +29 -0
- metadata +234 -0
@@ -0,0 +1,95 @@
|
|
1
|
+
=begin
|
2
|
+
|
3
|
+
= mysqlcmds.rb
|
4
|
+
|
5
|
+
*Copyright*:: (C) 2013 by Novu, LLC
|
6
|
+
*Author(s)*:: Tamara Temple <tamara.temple@novu.com>
|
7
|
+
*Since*:: 2013-05-01
|
8
|
+
*License*:: MIT
|
9
|
+
*Version*:: 0.0.1
|
10
|
+
|
11
|
+
== Description
|
12
|
+
|
13
|
+
Class to wrap mysql command line applications mysqldump, mysqladmin, and mysql.
|
14
|
+
|
15
|
+
=end
|
16
|
+
|
17
|
+
require 'methadone'
|
18
|
+
require 'RunIt'
|
19
|
+
|
20
|
+
module NewBackup
|
21
|
+
|
22
|
+
class MySqlCmds
|
23
|
+
|
24
|
+
include Methadone::CLILogging
|
25
|
+
|
26
|
+
attr_accessor :hostname, :username, :password, :database, :obfuscate_script
|
27
|
+
|
28
|
+
|
29
|
+
def initialize(hostname, username, password, database, obfuscate)
|
30
|
+
self.hostname=hostname.dup.to_s
|
31
|
+
self.username=username.dup.to_s
|
32
|
+
self.password=password.dup.to_s
|
33
|
+
self.database=database.dup.to_s
|
34
|
+
self.obfuscate_script=obfuscate.dup.to_s
|
35
|
+
|
36
|
+
@commands =
|
37
|
+
{:mysqldump => get_command('mysqldump'),
|
38
|
+
:mysql => get_command('mysql'),
|
39
|
+
:gzip => get_command('gzip')
|
40
|
+
}
|
41
|
+
|
42
|
+
debug "#{self.class}##{__method__}:#{__LINE__}: @commands: #{@commands}"
|
43
|
+
|
44
|
+
end
|
45
|
+
|
46
|
+
def dump(save_file)
|
47
|
+
cmd = []
|
48
|
+
cmd << @commands[:mysqldump]
|
49
|
+
cmd << "--host #{self.hostname}"
|
50
|
+
cmd << "--user=#{self.username}"
|
51
|
+
cmd << "--password=#{self.password}" unless self.password.nil? or self.password.empty?
|
52
|
+
cmd << self.database
|
53
|
+
cmd << "|"
|
54
|
+
cmd << @commands[:gzip]
|
55
|
+
cmd << ">"
|
56
|
+
cmd << save_file
|
57
|
+
|
58
|
+
debug "#{self.class}##{__method__}:#{__LINE__}: cmd = #{cmd.join(" ")}"
|
59
|
+
|
60
|
+
saver = RunIt.new cmd.join(" ")
|
61
|
+
saver.run
|
62
|
+
raise "#{self.class}#save error: #{saver.result.exitstatus}: #{saver.error.inspect}" unless saver.result.success?
|
63
|
+
debug "#{self.class}##{__method__}:#{__LINE__}: saver.output: #{saver.output}"
|
64
|
+
end
|
65
|
+
|
66
|
+
def obfuscate
|
67
|
+
|
68
|
+
cmd = []
|
69
|
+
cmd << @commands[:mysql]
|
70
|
+
cmd << "--host #{self.hostname}"
|
71
|
+
cmd << "--user #{self.username}"
|
72
|
+
cmd << "--password=#{self.password}" unless self.password.nil? or self.password.empty?
|
73
|
+
cmd << self.database
|
74
|
+
cmd << "<"
|
75
|
+
cmd << self.obfuscate_script
|
76
|
+
|
77
|
+
debug "#{self.class}##{__method__}:#{__LINE__}: cmd= #{cmd.join(" ")}"
|
78
|
+
obfuscator = RunIt.new cmd.join(" ")
|
79
|
+
obfuscator.run
|
80
|
+
raise "#{self.class}##{__method__}:#{__LINE__}: error: #{obfuscator.result.exitstatus}: #{obfuscator.error.inspect}" unless obfuscator.result.success?
|
81
|
+
debug "#{self.class}##{__method__}:#{__LINE__}: obfuscator.output: #{obfuscator.output}"
|
82
|
+
end
|
83
|
+
|
84
|
+
|
85
|
+
private
|
86
|
+
|
87
|
+
def get_command(command)
|
88
|
+
command_path = `which #{command}`.chomp
|
89
|
+
raise "#{command} not found!" if command_path.empty? || command_path.nil?
|
90
|
+
command_path
|
91
|
+
end
|
92
|
+
|
93
|
+
end
|
94
|
+
|
95
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
=begin
|
2
|
+
|
3
|
+
= new_backup/version.rb
|
4
|
+
|
5
|
+
*Copyright*:: (C) 2013 by Novu, LLC
|
6
|
+
*Author(s)*:: Tamara Temple <tamara.temple@novu.com>
|
7
|
+
|
8
|
+
=end
|
9
|
+
|
10
|
+
|
11
|
+
|
12
|
+
|
13
|
+
module NewBackup
|
14
|
+
VERSION = "1.0.1"
|
15
|
+
DESCRIPTION = "Backup an RDS database instance to S3, and create and obfuscated version"
|
16
|
+
SUMMARY = DESCRIPTION
|
17
|
+
end
|
data/new_backup.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
lib = File.expand_path('../lib', __FILE__)
|
2
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
3
|
+
require 'new_backup/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |gem|
|
6
|
+
gem.name = "new_backup"
|
7
|
+
gem.version = NewBackup::VERSION
|
8
|
+
gem.authors = ["Tamara Temple"]
|
9
|
+
gem.email = ["tamouse@gmail.com"]
|
10
|
+
gem.description = NewBackup::DESCRIPTION
|
11
|
+
gem.summary = NewBackup::SUMMARY
|
12
|
+
gem.homepage = ""
|
13
|
+
|
14
|
+
gem.files = `git ls-files`.split($/)
|
15
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
16
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
17
|
+
gem.require_paths = ["lib"]
|
18
|
+
gem.add_development_dependency('rdoc')
|
19
|
+
gem.add_development_dependency('aruba')
|
20
|
+
gem.add_development_dependency('rake', '~> 0.9.2')
|
21
|
+
gem.add_dependency('methadone', '~> 1.2.6')
|
22
|
+
gem.add_dependency('RunIt')
|
23
|
+
gem.add_dependency('fog')
|
24
|
+
gem.add_dependency('dogapi')
|
25
|
+
gem.add_development_dependency('rspec')
|
26
|
+
gem.add_development_dependency('debugger')
|
27
|
+
end
|
data/sample-config.yml
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# Sample Config file for the RDS->S3 backup script
|
2
|
+
fog_timeout: 3600
|
3
|
+
dump_directory: /tmp
|
4
|
+
dump_ttl: 20
|
5
|
+
aws_region: us-west-1
|
6
|
+
db_instance_type: db.t1.micro
|
7
|
+
timestamp_format: "%FT%T-%X"
|
8
|
+
rds_instance_id: myrds
|
9
|
+
db_subnet_group_name: yolo
|
10
|
+
backup_bucket: raw_backups
|
11
|
+
s3_bucket: clean_backups
|
12
|
+
s3_prefix: felix
|
13
|
+
aws_s3_region: outermars
|
14
|
+
mysql_username: despereaux
|
15
|
+
mysql_database: gollywogs
|
16
|
+
mysql_password: nutz
|
17
|
+
obfuscate_script: nolocontendre
|
18
|
+
datadog_apikey: youdthink
|
19
|
+
aws_access_key_id: bogus aws key
|
20
|
+
aws_secret_access_key: bogus aws secret
|
@@ -0,0 +1,31 @@
|
|
1
|
+
=begin
|
2
|
+
|
3
|
+
= datadog_spec.rb
|
4
|
+
|
5
|
+
*Copyright*:: (C) 2013 by Novu, LLC
|
6
|
+
*Author(s)*:: Tamara Temple <tamara.temple@novu.com>
|
7
|
+
*Since*:: 2013-05-02
|
8
|
+
*License*:: GPLv3
|
9
|
+
*Version*:: 0.0.1
|
10
|
+
|
11
|
+
=end
|
12
|
+
|
13
|
+
require 'spec_helper'
|
14
|
+
require 'new_backup/datadog'
|
15
|
+
|
16
|
+
|
17
|
+
module NewBackup
|
18
|
+
|
19
|
+
describe DataDog do
|
20
|
+
|
21
|
+
it "#api_key=" do
|
22
|
+
NewBackup::DataDog.api_key= '12345'
|
23
|
+
NewBackup::DataDog.api_key.should == '12345'
|
24
|
+
end
|
25
|
+
|
26
|
+
|
27
|
+
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
end
|
data/spec/main_spec.rb
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
=begin
|
2
|
+
|
3
|
+
= new_backup_spec.rb
|
4
|
+
|
5
|
+
*Copyright*:: (C) 2013 by Novu, LLC
|
6
|
+
*Author(s)*:: Tamara Temple <tamara.temple@novu.com>
|
7
|
+
*Since*:: 2013-05-01
|
8
|
+
*License*:: GPLv3
|
9
|
+
*Version*:: 0.0.1
|
10
|
+
|
11
|
+
== Description
|
12
|
+
|
13
|
+
Test base library module.
|
14
|
+
|
15
|
+
=end
|
16
|
+
|
17
|
+
require 'spec_helper'
|
18
|
+
require 'new_backup/main'
|
19
|
+
|
20
|
+
def check_keys(h,klist)
|
21
|
+
(h.keys <=> klist) == 0
|
22
|
+
end
|
23
|
+
|
24
|
+
module NewBackup
|
25
|
+
|
26
|
+
describe Main do
|
27
|
+
|
28
|
+
describe "processing options and initializing" do
|
29
|
+
let(:dogger) {double('dogger')}
|
30
|
+
|
31
|
+
it "no options specified" do
|
32
|
+
expect {NewBackup::Main.new()}.to raise_error(RuntimeError)
|
33
|
+
end
|
34
|
+
|
35
|
+
describe "full options" do
|
36
|
+
let(:options) {{
|
37
|
+
'fog_timeout' => 3600,
|
38
|
+
'dump_directory' => '/tmp',
|
39
|
+
'dump_ttl' => 20,
|
40
|
+
'aws_region' => 'us-west-1',
|
41
|
+
'db_instance_type' => 'db.t1.micro',
|
42
|
+
'timestamp_format' => "%FT%T-%Z",
|
43
|
+
'rds_instance_id' => 'myrds',
|
44
|
+
'db_subnet_group_name' => 'yolo',
|
45
|
+
'backup_bucket' => 'raw_backups',
|
46
|
+
's3_bucket' => 'clean_backups',
|
47
|
+
's3_prefix' => 'felix',
|
48
|
+
'aws_s3_region' => 'outermars',
|
49
|
+
'mysql_username' => 'despereaux',
|
50
|
+
'mysql_database' => 'gollywogs',
|
51
|
+
'mysql_password' => 'nutz',
|
52
|
+
'obfuscate_script' => 'nolocontendre',
|
53
|
+
'datadog_apikey' => 'youdthink',
|
54
|
+
'aws_access_key_id' => 'bogus aws key',
|
55
|
+
'aws_secret_access_key' => 'bogus aws secret'
|
56
|
+
}}
|
57
|
+
let(:main) { NewBackup::Main.new(options) }
|
58
|
+
let(:actual_options) { main.options }
|
59
|
+
|
60
|
+
it {main.should be_a(NewBackup::Main)}
|
61
|
+
it {check_keys(actual_options, %w{fog aws rds s3 mysql datadog dump_directory timestamp debug nords nos3}.map(&:to_sym)).should be_true}
|
62
|
+
it {check_keys(actual_options[:fog], %w{timeout}.map(&:to_sym)).should be_true}
|
63
|
+
it {check_keys(actual_options[:aws], %w{access_key secret_key region}.map(&:to_sym)).should be_true}
|
64
|
+
it {check_keys(actual_options[:rds], %w{instance_id subnet_group instance_type}.map(&:to_sym)).should be_true}
|
65
|
+
it {check_keys(actual_options[:s3], %w{raw_bucket clean_bucket prefix region dump_ttl}.map(&:to_sym)).should be_true}
|
66
|
+
it {check_keys(actual_options[:mysql], %w{username password database obfuscate_script}.map(&:to_sym)).should be_true}
|
67
|
+
it {check_keys(actual_options[:datadog], %w{api_key}.map(&:to_sym)).should be_true}
|
68
|
+
|
69
|
+
it { main.should respond_to(:run) }
|
70
|
+
|
71
|
+
end
|
72
|
+
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
data/spec/mockbucket.rb
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
=begin
|
2
|
+
|
3
|
+
= mockbucket.rb
|
4
|
+
|
5
|
+
*Copyright*:: (C) 2013 by Novu, LLC
|
6
|
+
*Author(s)*:: Tamara Temple <tamara.temple@novu.com>
|
7
|
+
*Since*:: 2013-05-14
|
8
|
+
*License*:: GPLv3
|
9
|
+
*Version*:: 0.0.1
|
10
|
+
|
11
|
+
== Description
|
12
|
+
|
13
|
+
=end
|
14
|
+
|
15
|
+
class MockFile
|
16
|
+
attr_accessor :name, :last_modified, :key, :directory
|
17
|
+
def initialize(n,d,dir)
|
18
|
+
self.name = n.dup
|
19
|
+
self.key = self.name
|
20
|
+
self.last_modified = d.dup
|
21
|
+
self.directory = dir
|
22
|
+
end
|
23
|
+
|
24
|
+
def destroy
|
25
|
+
self.name = nil
|
26
|
+
self.key = nil
|
27
|
+
self.last_modified = nil
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
class MockBucket
|
32
|
+
attr_accessor :bucket_contents, :name, :key
|
33
|
+
def initialize(n=1, name)
|
34
|
+
self.bucket_contents = (1..n).to_a.map{|i| MockFile.new("file-#{i}", Time.new(2012, 6, (i+1), 0, 0, 0), self) }
|
35
|
+
self.name = name.dup
|
36
|
+
self.key = self.name
|
37
|
+
end
|
38
|
+
|
39
|
+
def files
|
40
|
+
self
|
41
|
+
end
|
42
|
+
|
43
|
+
def create(options={})
|
44
|
+
self.bucket_contents << MockFile.new(options[:key], Time.now, self)
|
45
|
+
end
|
46
|
+
|
47
|
+
def all(options={})
|
48
|
+
@bucket_contents.reject {|f| f.name.nil?}
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
|
54
|
+
class MockConnection
|
55
|
+
attr_accessor :bucket_list, :name, :key
|
56
|
+
def initialize(n=1, name)
|
57
|
+
self.name = name.dup
|
58
|
+
self.key = self.name
|
59
|
+
if n > 0
|
60
|
+
self.bucket_list = (1..n).to_a.map{|i| MockBucket.new(10, "bucket-#{i}")}
|
61
|
+
else
|
62
|
+
self.bucket_list = []
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
|
67
|
+
def directories
|
68
|
+
@bucket_list
|
69
|
+
end
|
70
|
+
|
71
|
+
def get(name)
|
72
|
+
@bucket_list.select {|bucket| bucket.name == name}.first
|
73
|
+
end
|
74
|
+
|
75
|
+
def create(name)
|
76
|
+
@bucket_list << MockBucket.new(10, name)
|
77
|
+
end
|
78
|
+
|
79
|
+
def destroy(name)
|
80
|
+
@bucket_list.reject! {|bucket| bucket.name == name}
|
81
|
+
end
|
82
|
+
|
83
|
+
end
|
data/spec/myrds_spec.rb
ADDED
@@ -0,0 +1,131 @@
|
|
1
|
+
=begin
|
2
|
+
|
3
|
+
= myrds_spec.rb
|
4
|
+
|
5
|
+
*Copyright*:: (C) 2013 by Novu, LLC
|
6
|
+
*Author(s)*:: Tamara Temple <tamara.temple@novu.com>
|
7
|
+
*Since*:: 2013-05-01
|
8
|
+
*License*:: GPLv3
|
9
|
+
*Version*:: 0.0.1
|
10
|
+
|
11
|
+
== Description
|
12
|
+
|
13
|
+
Test myrds.rb
|
14
|
+
|
15
|
+
=end
|
16
|
+
|
17
|
+
require 'spec_helper'
|
18
|
+
require 'new_backup/myrds.rb'
|
19
|
+
require 'fog'
|
20
|
+
require 'date'
|
21
|
+
|
22
|
+
module NewBackup
|
23
|
+
|
24
|
+
describe MyRds do
|
25
|
+
|
26
|
+
it {NewBackup::MyRds.should respond_to(:new)}
|
27
|
+
|
28
|
+
describe "Operations" do
|
29
|
+
|
30
|
+
|
31
|
+
it "#connect" do
|
32
|
+
Fog.mock!
|
33
|
+
options = {
|
34
|
+
:aws => {
|
35
|
+
:access_key => 'bogus access key',
|
36
|
+
:secret_key => 'bogus secret key',
|
37
|
+
:rds_region => 'us-east-1'},
|
38
|
+
:fog => {
|
39
|
+
:timeout => 10
|
40
|
+
},
|
41
|
+
:timestamp => DateTime.now.iso8601
|
42
|
+
}
|
43
|
+
|
44
|
+
NewBackup::MyRds.new(options).connect do |connection|
|
45
|
+
connection.should_not be_nil
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
it "#retrieve_snapshot" do
|
50
|
+
|
51
|
+
Fog.mock!
|
52
|
+
options = {
|
53
|
+
:rds => {
|
54
|
+
:instance_id => 'tamtest'
|
55
|
+
},
|
56
|
+
:aws => {
|
57
|
+
:access_key => 'bogus access key',
|
58
|
+
:secret_key => 'bogus secret key',
|
59
|
+
:rds_region => 'us-east-1'},
|
60
|
+
:fog => {
|
61
|
+
:timeout => 10},
|
62
|
+
:timestamp => DateTime.now.iso8601
|
63
|
+
}
|
64
|
+
|
65
|
+
rds_server = double('rds_server')
|
66
|
+
snapshot = double('snapshot')
|
67
|
+
rds_server.stub_chain(:snapshots, :new, :save)
|
68
|
+
rds_server.stub_chain(:snapshots, :get) {snapshot}
|
69
|
+
snapshot.stub(:wait_for)
|
70
|
+
snapshot.stub(:destroy)
|
71
|
+
|
72
|
+
NewBackup::MyRds.new(options).retrieve_snapshot(rds_server) do |snapshot|
|
73
|
+
snapshot.should_not be_nil
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
|
78
|
+
it "#restore_db" do
|
79
|
+
|
80
|
+
Fog.mock!
|
81
|
+
options = {
|
82
|
+
:rds => {
|
83
|
+
:instance_id => 'tamtest',
|
84
|
+
:subnet_group => '',
|
85
|
+
:instance_type => 'db.t1.micro'
|
86
|
+
},
|
87
|
+
:aws => {
|
88
|
+
:access_key => 'bogus access key',
|
89
|
+
:secret_key => 'bogus secret key',
|
90
|
+
:rds_region => 'us-east-1'},
|
91
|
+
:mysql => {
|
92
|
+
:database => 'qadb',
|
93
|
+
:username => 'root',
|
94
|
+
:password => 'aaaa',
|
95
|
+
:obfuscate_script => 'zzzz'
|
96
|
+
},
|
97
|
+
:fog => {
|
98
|
+
:timeout => 10
|
99
|
+
},
|
100
|
+
:timestamp => DateTime.now.iso8601
|
101
|
+
}
|
102
|
+
connection = double('connection')
|
103
|
+
snapshot = double('snapshot')
|
104
|
+
backup_server = double('backup_server')
|
105
|
+
connection.stub(:restore_db_instance_from_db_snapshot)
|
106
|
+
connection.stub_chain(:servers, :get) {backup_server}
|
107
|
+
snapshot.stub(:id)
|
108
|
+
backup_server.stub_chain(:endpoint, :[]).with("Address").and_return("localhost")
|
109
|
+
backup_server.stub(:wait_for)
|
110
|
+
backup_server.stub(:destroy)
|
111
|
+
|
112
|
+
NewBackup::MyRds.new(options).restore_db(connection, snapshot) do |db|
|
113
|
+
db.should_not be_nil
|
114
|
+
end
|
115
|
+
|
116
|
+
end
|
117
|
+
|
118
|
+
end
|
119
|
+
|
120
|
+
|
121
|
+
describe "#snap_name" do
|
122
|
+
let(:timestamp) {DateTime.now.iso8601}
|
123
|
+
it {NewBackup::MyRds.new(:timestamp => timestamp).snap_name.should == "s3-dump-snap-#{timestamp}" }
|
124
|
+
end
|
125
|
+
|
126
|
+
|
127
|
+
|
128
|
+
|
129
|
+
end
|
130
|
+
|
131
|
+
end
|