s3mybackup 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.rdoc ADDED
@@ -0,0 +1,6 @@
1
+ = s3backup
2
+
3
+ Describe your project here
4
+
5
+ :include:s3mybackup.rdoc
6
+
data/bin/s3mybackup ADDED
@@ -0,0 +1,231 @@
1
+ #!/usr/bin/env ruby
2
+ require 'gli'
3
+ require "rubygems"
4
+ require "aws-sdk"
5
+ require "yaml"
6
+ require "fileutils"
7
+ require "socket"
8
+ require File.join(File.dirname(__FILE__), "../lib", 's3mybackup')
9
+
10
+ include GLI::App
11
+
12
+ program_desc 'Tool for backing up and restoring mysql databases to and from S3.'
13
+
14
+ version = S3mybackup::VERSION
15
+
16
+ config_file = File.join(".","config.yml")
17
+
18
+ unless File.exist?(config_file) then
19
+ raise "File #{config_file} does not exist."
20
+ end
21
+
22
+ defaults = YAML.load(File.read(config_file))
23
+
24
+ unless defaults.kind_of?(Hash) then
25
+ raise "config.yml is formatted incorrectly. Please use the following format: \naccess_key_id: YOUR_ACCESS_KEY_ID\nsecret_access_key: YOUR_SECRET_ACCESS_KEY"
26
+ end
27
+
28
+ desc 'User'
29
+ default_value 'backup'
30
+ arg_name 'Backup user'
31
+ flag [:u, :user]
32
+
33
+ desc 'Password'
34
+ arg_name 'Password'
35
+ flag [:p, :password]
36
+
37
+ desc 'Directory where mysql writes binary logs. log-bin = in my.cnf'
38
+ default_value defaults[:mysql_bin_log_dir]
39
+ arg_name 'mysql_bin_log_dir'
40
+ flag [:l, :mysql_bin_log_dir]
41
+
42
+ desc 'Temporary directory where files are stored before writing to and after download from S3'
43
+ default_value defaults[:temp_dir]
44
+ arg_name 'mysql_bin_log_dir'
45
+ flag [:t, :temp_dir]
46
+
47
+ desc 'Access key S3'
48
+ default_value defaults[:access_key_id]
49
+ arg_name 'access_key_id'
50
+ flag [:a, :access_key_id]
51
+
52
+ desc 'Secret access key S3'
53
+ default_value defaults[:secret_access_key]
54
+ arg_name 'secret_access_key'
55
+ flag [:s, :secret_access_key]
56
+
57
+ desc 'Secret access key S3'
58
+ default_value defaults[:s3_endpoint]
59
+ arg_name 's3_endpoint'
60
+ flag [:e, :s3_endpoint]
61
+
62
+ desc 'Bucket for backups'
63
+ arg_name 'Bucket'
64
+ flag [:b, :bucket]
65
+
66
+ desc 'Creates a full backup to S3'
67
+ arg_name 'Describe arguments to full here'
68
+
69
+ command :full do |c|
70
+
71
+ c.action do |global_options, options, args|
72
+
73
+ bucket = get_bucket(global_options)
74
+
75
+ database = args[0]
76
+
77
+ database_dir = get_database_dir(database)
78
+
79
+ dump_file_name = get_full_backup_file_name
80
+
81
+ # delete all incremental backup files
82
+ bucket.objects.with_prefix(database_dir).delete_if { |o| o.key.include?("-bin.") }
83
+
84
+ user = global_options[:user]
85
+ password = global_options[:password]
86
+
87
+
88
+ # assumes the bucket's empty
89
+ dump_file = "#{@temp_dir}/#{dump_file_name}"
90
+
91
+ cmd = "mysqldump --quick --single-transaction --create-options -u #{user} --flush-logs --master-data=2 --delete-master-logs"
92
+ cmd += " -p'#{password}'" unless password.nil?
93
+ cmd += " #{database} | gzip > #{dump_file}"
94
+ run_system(cmd)
95
+
96
+ bucket.objects.create("#{database_dir}/#{dump_file_name}", open(dump_file))
97
+ end
98
+ end
99
+
100
+ desc 'Executes incremental backup to S3'
101
+ arg_name 'database to backup'
102
+ command :inc do |c|
103
+ c.action do |global_options, options, args|
104
+ user = global_options[:user]
105
+ password = global_options[:password]
106
+ execute_sql "flush logs", user, password
107
+ logs = Dir.glob("#{@mysql_bin_log_dir}/*-bin.[0-9]*").sort
108
+ logs_to_archive = logs[0..-2] # all logs except the last
109
+
110
+ bucket = get_bucket global_options
111
+
112
+ logs_to_archive.each do |log|
113
+ bucket.objects.create("#{get_database_dir(args[0])}/#{File.basename(log)}", open(log))
114
+ end
115
+ if logs[-1] then
116
+ execute_sql "purge master logs to '#{File.basename(logs[-1])}'", user, password
117
+ end
118
+ end
119
+ end
120
+
121
+ desc 'Describe restore here'
122
+ arg_name 'Describe arguments to restore here'
123
+ command :restore do |c|
124
+ c.desc 'Restore the binary logs'
125
+ c.switch [:l, :logs]
126
+
127
+ c.desc 'IP address of server without the dots'
128
+ c.arg_name 'Ip'
129
+ c.default_value get_ip
130
+ c.flag [:i, :ip]
131
+
132
+ c.desc 'Time of the backup to restore'
133
+ c.arg_name 'Time'
134
+ c.flag [:t, :time]
135
+ c.action do |global_options, options, args|
136
+ time = options[:time]
137
+
138
+ unless time then
139
+ raise "Option time is required"
140
+ end
141
+
142
+ user = global_options[:user]
143
+ password = global_options[:password]
144
+
145
+ ip = options[:ip]
146
+
147
+ bucket = get_bucket global_options
148
+
149
+ from_database = args[0]
150
+ to_database = args[1]
151
+
152
+ unless to_database then
153
+ to_database = from_database
154
+ end
155
+
156
+ database_dir = get_database_dir(from_database, ip)
157
+
158
+ full_backup_file_name = get_full_backup_file_name(time)
159
+
160
+ backup_file_key = "#{database_dir}/#{full_backup_file_name}"
161
+ unless bucket.objects[backup_file_key].exists? then
162
+ raise "Dump #{backup_file_key} does not not exist in bucket #{bucket.name}"
163
+ end
164
+ retrieve_file(bucket, database_dir, full_backup_file_name)
165
+
166
+ # restore the dump file
167
+ cmd = "gunzip -c #{@temp_dir}/#{full_backup_file_name} | mysql -u #{user} "
168
+ cmd += " -p'#{password}' " unless password.nil?
169
+ cmd += " #{to_database}"
170
+ run_system cmd
171
+
172
+ if options[:logs] then
173
+
174
+ binary_log_objects = bucket.objects.with_prefix(database_dir).select { |obj| obj.key.match(/.*-bin\.[0-9]{6}/) }
175
+ binary_log_objects.each do |o|
176
+ File.open("#{@temp_dir}/#{File.basename(o.key)}", 'w') do |f|
177
+ o.read do |chunk|
178
+ f.write(chunk)
179
+ end
180
+ end
181
+ end
182
+
183
+
184
+ logs = Dir.glob("#{@temp_dir}/*-bin.[0-9]*").sort
185
+
186
+ # restore the binary log files
187
+ logs.each do |log|
188
+ # The following will be executed for each binary log file
189
+ cmd = "mysqlbinlog --database=#{from_database} #{log} | mysql -u #{user} "
190
+ cmd += " -p'#{password}' " unless password.nil?
191
+ cmd += " #{to_database}"
192
+ run_system cmd
193
+ end
194
+ end
195
+ end
196
+ end
197
+
198
+ pre do |global, command, options, args|
199
+ if args.empty? then
200
+ raise "Database name is required."
201
+ end
202
+
203
+
204
+
205
+
206
+
207
+ @mysql_bin_log_dir = global[:mysql_bin_log_dir]
208
+ @temp_dir = global[:temp_dir]
209
+
210
+ AWS.config({
211
+ :access_key_id => global[:access_key_id],
212
+ :secret_access_key => global[:secret_access_key],
213
+ :s3_endpoint => global[:s3_endpoint]
214
+ })
215
+
216
+ FileUtils.mkdir_p @temp_dir
217
+
218
+ true
219
+ end
220
+
221
+ post do |global, command, options, args|
222
+ FileUtils.rm_rf(@temp_dir)
223
+ end
224
+
225
+ on_error do |exception|
226
+ # Error logic here
227
+ # return false to skip default error handling
228
+ true
229
+ end
230
+
231
+ exit run(ARGV)
@@ -0,0 +1,3 @@
1
+ module S3mybackup
2
+ VERSION = '0.0.2'
3
+ end
data/lib/s3mybackup.rb ADDED
@@ -0,0 +1,47 @@
1
+ require 's3mybackup/version.rb'
2
+
3
+ def run_system(command)
4
+ result = system(command)
5
+ raise("error, process exited with status #{$?.exitstatus}") unless result
6
+ end
7
+
8
+ def execute_sql(sql,user,password)
9
+ cmd = "mysql -u #{user} -e \"#{sql}\""
10
+ cmd += " -p'#{password}' " unless password.nil?
11
+ run_system cmd
12
+ end
13
+
14
+ def get_bucket(global_options)
15
+ s3 = AWS::S3.new
16
+
17
+ bucket_name = global_options[:bucket]
18
+ bucket = s3.buckets[bucket_name]
19
+
20
+ if not bucket.exists? then
21
+ bucket = s3.buckets.create(bucket_name)
22
+ end
23
+ bucket
24
+ end
25
+
26
+ def get_ip
27
+ IPSocket.getaddress(Socket.gethostname).gsub!(/\./, "")
28
+ end
29
+
30
+ def get_database_dir(database,ip = get_ip)
31
+ database_dir = "#{ip}/#{database}"
32
+ end
33
+
34
+ def get_full_backup_file_name(time_string = (Time.now).strftime('%Y%m%d%H%M%S'))
35
+ "#{time_string}dump.sql.gz"
36
+ end
37
+
38
+ def retrieve_file(bucket,database_dir,file_name)
39
+
40
+ full_dump = bucket.objects["#{database_dir}/#{file_name}"]
41
+
42
+ File.open("#{@temp_dir}/#{file_name}", 'w') do |f|
43
+ full_dump.read do |chunk|
44
+ f.write(chunk)
45
+ end
46
+ end
47
+ end
data/s3mybackup.rdoc ADDED
@@ -0,0 +1,5 @@
1
+ = s3backup
2
+
3
+ Generate this with
4
+ s3backup rdoc
5
+ After you have described your command line interface
metadata ADDED
@@ -0,0 +1,150 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: s3mybackup
3
+ version: !ruby/object:Gem::Version
4
+ hash: 27
5
+ prerelease:
6
+ segments:
7
+ - 0
8
+ - 0
9
+ - 2
10
+ version: 0.0.2
11
+ platform: ruby
12
+ authors:
13
+ - Joris Wijlens
14
+ autorequire:
15
+ bindir: bin
16
+ cert_chain: []
17
+
18
+ date: 2012-12-30 00:00:00 +01:00
19
+ default_executable:
20
+ dependencies:
21
+ - !ruby/object:Gem::Dependency
22
+ name: rake
23
+ prerelease: false
24
+ requirement: &id001 !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ hash: 3
30
+ segments:
31
+ - 0
32
+ version: "0"
33
+ type: :development
34
+ version_requirements: *id001
35
+ - !ruby/object:Gem::Dependency
36
+ name: rdoc
37
+ prerelease: false
38
+ requirement: &id002 !ruby/object:Gem::Requirement
39
+ none: false
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ hash: 3
44
+ segments:
45
+ - 0
46
+ version: "0"
47
+ type: :development
48
+ version_requirements: *id002
49
+ - !ruby/object:Gem::Dependency
50
+ name: aruba
51
+ prerelease: false
52
+ requirement: &id003 !ruby/object:Gem::Requirement
53
+ none: false
54
+ requirements:
55
+ - - ">="
56
+ - !ruby/object:Gem::Version
57
+ hash: 3
58
+ segments:
59
+ - 0
60
+ version: "0"
61
+ type: :development
62
+ version_requirements: *id003
63
+ - !ruby/object:Gem::Dependency
64
+ name: gli
65
+ prerelease: false
66
+ requirement: &id004 !ruby/object:Gem::Requirement
67
+ none: false
68
+ requirements:
69
+ - - "="
70
+ - !ruby/object:Gem::Version
71
+ hash: 29
72
+ segments:
73
+ - 2
74
+ - 5
75
+ - 3
76
+ version: 2.5.3
77
+ type: :runtime
78
+ version_requirements: *id004
79
+ - !ruby/object:Gem::Dependency
80
+ name: aws-sdk
81
+ prerelease: false
82
+ requirement: &id005 !ruby/object:Gem::Requirement
83
+ none: false
84
+ requirements:
85
+ - - "="
86
+ - !ruby/object:Gem::Version
87
+ hash: 9
88
+ segments:
89
+ - 1
90
+ - 7
91
+ - 1
92
+ version: 1.7.1
93
+ type: :runtime
94
+ version_requirements: *id005
95
+ description: " Options can be configured in a file config.yml located in the directory from where command is executed.\n"
96
+ email: joris@smartworkx.com
97
+ executables:
98
+ - s3mybackup
99
+ extensions: []
100
+
101
+ extra_rdoc_files:
102
+ - README.rdoc
103
+ - s3mybackup.rdoc
104
+ files:
105
+ - bin/s3mybackup
106
+ - lib/s3mybackup/version.rb
107
+ - lib/s3mybackup.rb
108
+ - README.rdoc
109
+ - s3mybackup.rdoc
110
+ has_rdoc: true
111
+ homepage: http://www.smartworkx.nl
112
+ licenses: []
113
+
114
+ post_install_message:
115
+ rdoc_options:
116
+ - --title
117
+ - s3mybackup
118
+ - --main
119
+ - README.rdoc
120
+ - -ri
121
+ require_paths:
122
+ - lib
123
+ - lib
124
+ required_ruby_version: !ruby/object:Gem::Requirement
125
+ none: false
126
+ requirements:
127
+ - - ">="
128
+ - !ruby/object:Gem::Version
129
+ hash: 3
130
+ segments:
131
+ - 0
132
+ version: "0"
133
+ required_rubygems_version: !ruby/object:Gem::Requirement
134
+ none: false
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ hash: 3
139
+ segments:
140
+ - 0
141
+ version: "0"
142
+ requirements: []
143
+
144
+ rubyforge_project:
145
+ rubygems_version: 1.4.2
146
+ signing_key:
147
+ specification_version: 3
148
+ summary: Command suite for backing up MySQL databases to S3
149
+ test_files: []
150
+