mysql_truck 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -0
- data/Gemfile +4 -0
- data/Rakefile +1 -0
- data/bin/mysql_truck +90 -0
- data/lib/mysql_truck.rb +240 -0
- data/lib/mysql_truck/version.rb +3 -0
- data/mysql_truck.gemspec +20 -0
- metadata +67 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/bin/mysql_truck
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
require 'mysql_truck'
|
5
|
+
require 'yaml'
|
6
|
+
|
7
|
+
options = {}
|
8
|
+
|
9
|
+
parser = OptionParser.new do |opts|
|
10
|
+
opts.banner = "Usage: #{File.basename(__FILE__)} (dump|load) [options]"
|
11
|
+
|
12
|
+
opts.separator ""
|
13
|
+
opts.separator "Required Options:"
|
14
|
+
|
15
|
+
opts.on("-c", "--config FILE", "Configuration yaml file") do |file|
|
16
|
+
options[:config] = YAML.load_file(Pathname.new(file).expand_path)
|
17
|
+
end
|
18
|
+
|
19
|
+
opts.separator ""
|
20
|
+
opts.separator "Or specify options individually:"
|
21
|
+
opts.separator "(options here override options from the config if specified)"
|
22
|
+
|
23
|
+
opts.on("-H", "--host HOST", "Database host") do |host|
|
24
|
+
options[:host] = host
|
25
|
+
end
|
26
|
+
|
27
|
+
opts.on("-u", "--user USERNAME", "Database user") do |user|
|
28
|
+
options[:username] = user
|
29
|
+
end
|
30
|
+
|
31
|
+
opts.on("-p", "--password PASSWORD", "Database password") do |password|
|
32
|
+
options[:password] = password
|
33
|
+
end
|
34
|
+
|
35
|
+
opts.on("-d", "--database DATABASE_NAME", "Database name") do |db|
|
36
|
+
options[:database] = db
|
37
|
+
end
|
38
|
+
|
39
|
+
opts.on("-a", "--access-key KEY", "S3 Access Key") do |key|
|
40
|
+
options[:s3_access_key] = key
|
41
|
+
end
|
42
|
+
|
43
|
+
opts.on("-s", "--secret-access-key KEY", "S3 Secret Acess Key") do |key|
|
44
|
+
options[:s3_secret_access_key] = key
|
45
|
+
end
|
46
|
+
|
47
|
+
opts.on("-t", "--skip-tables TABLES",
|
48
|
+
"List of tables to skip separated by commas.") do |tables|
|
49
|
+
options[:skip_tables] = tables.split(",")
|
50
|
+
end
|
51
|
+
|
52
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
53
|
+
puts opts
|
54
|
+
exit
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
begin
|
59
|
+
parser.parse!(ARGV)
|
60
|
+
|
61
|
+
if options[:config]
|
62
|
+
config = {}
|
63
|
+
options.delete(:config).each do |k,v|
|
64
|
+
config[k.intern] = v
|
65
|
+
end
|
66
|
+
|
67
|
+
options = config.merge(options)
|
68
|
+
end
|
69
|
+
|
70
|
+
missing_opts = []
|
71
|
+
[ :s3_access_key, :s3_secret_access_key,
|
72
|
+
:host, :username, :database
|
73
|
+
].each do |opt|
|
74
|
+
missing_opts << opt unless options.include?(opt)
|
75
|
+
end
|
76
|
+
|
77
|
+
if missing_opts.size > 0
|
78
|
+
puts "Missing the following options:"
|
79
|
+
missing_opts.each { |o| puts " * #{o}" }
|
80
|
+
puts
|
81
|
+
puts parser
|
82
|
+
exit
|
83
|
+
end
|
84
|
+
rescue OptionParser::ParseError => e
|
85
|
+
puts "\nPlease ensure you have the correct options\n\n"
|
86
|
+
puts parser
|
87
|
+
exit
|
88
|
+
end
|
89
|
+
|
90
|
+
MysqlTruck.run(ARGV.first, options)
|
data/lib/mysql_truck.rb
ADDED
@@ -0,0 +1,240 @@
|
|
1
|
+
require "mysql_truck/version"
|
2
|
+
require "right_aws"
|
3
|
+
require 'fileutils'
|
4
|
+
require 'pathname'
|
5
|
+
|
6
|
+
# MysqlTruck
|
7
|
+
#
|
8
|
+
# MySQL backup tool that stores backups in S3.
|
9
|
+
#
|
10
|
+
# Requires server to have right_aws installed
|
11
|
+
module MysqlTruck
|
12
|
+
include FileUtils
|
13
|
+
|
14
|
+
def self.run(action, config)
|
15
|
+
case action.intern
|
16
|
+
when :dump
|
17
|
+
Dumper.new(config).dump
|
18
|
+
when :load
|
19
|
+
Loader.new(config).load_latest
|
20
|
+
else
|
21
|
+
puts "Unknown action #{action}"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
module Helper
|
26
|
+
include FileUtils
|
27
|
+
|
28
|
+
def config
|
29
|
+
@config
|
30
|
+
end
|
31
|
+
|
32
|
+
def initialize_s3
|
33
|
+
@s3 = RightAws::S3.new(
|
34
|
+
config[:s3_access_key],
|
35
|
+
config[:s3_secret_access_key])
|
36
|
+
@bucket = @s3.bucket("8tr.db_backups")
|
37
|
+
end
|
38
|
+
|
39
|
+
def db_connection_options
|
40
|
+
opts = %Q[ -u #{config[:username]} ]
|
41
|
+
opts += %Q[ -p"#{config[:password]}" ] unless config[:password].nil?
|
42
|
+
opts += %Q[ -h #{config[:host]} --default-character-set=utf8 ]
|
43
|
+
opts += %Q[ #{config[:database]} ]
|
44
|
+
opts
|
45
|
+
end
|
46
|
+
|
47
|
+
def csv_options
|
48
|
+
" --fields-enclosed-by=\\\" --fields-terminated-by=, "
|
49
|
+
end
|
50
|
+
|
51
|
+
def tmp_path
|
52
|
+
unless @tmp_path
|
53
|
+
@tmp_path = config[:tmp_dir] || Pathname.new("/data/s3backup")
|
54
|
+
@tmp_path = @tmp_path.join(@time.to_i.to_s) if @time # Only set with the Dumper class
|
55
|
+
end
|
56
|
+
|
57
|
+
@tmp_path
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
|
62
|
+
class Dumper
|
63
|
+
include FileUtils
|
64
|
+
include Helper
|
65
|
+
|
66
|
+
def initialize(config)
|
67
|
+
@config = config
|
68
|
+
@time = Time.now
|
69
|
+
|
70
|
+
initialize_s3
|
71
|
+
initialize_directories
|
72
|
+
end
|
73
|
+
|
74
|
+
def dump
|
75
|
+
dump_data
|
76
|
+
upload
|
77
|
+
|
78
|
+
ensure
|
79
|
+
# rm_r tmp_path, :force => true
|
80
|
+
end
|
81
|
+
|
82
|
+
def dump_data
|
83
|
+
tables.each do |table|
|
84
|
+
schema_file = tmp_path.join("#{table}.sql")
|
85
|
+
csv_file = tmp_path.join("#{table}.txt")
|
86
|
+
puts "Dumping #{table}."
|
87
|
+
|
88
|
+
# This command creates a table_name.sql and a table_name.txt file
|
89
|
+
cmd = "mysqldump --quick -T #{tmp_path} "
|
90
|
+
cmd += csv_options
|
91
|
+
cmd += "#{db_connection_options} #{table}"
|
92
|
+
puts cmd
|
93
|
+
`#{cmd}`
|
94
|
+
|
95
|
+
path, file = csv_file.split
|
96
|
+
csv_file = path.join("#{file.basename(".txt")}.csv")
|
97
|
+
mv path.join(file), csv_file
|
98
|
+
|
99
|
+
puts "gziping #{schema_file}."
|
100
|
+
`gzip #{schema_file}`
|
101
|
+
|
102
|
+
puts "gziping #{csv_file}."
|
103
|
+
`gzip #{csv_file}`
|
104
|
+
|
105
|
+
puts "#{table} dumped.\n\n"
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def upload
|
110
|
+
Dir["#{tmp_path}/*"].each do |file|
|
111
|
+
upload_file file
|
112
|
+
end
|
113
|
+
puts "Finished uploading backups."
|
114
|
+
end
|
115
|
+
|
116
|
+
private
|
117
|
+
|
118
|
+
def initialize_directories
|
119
|
+
mkdir_p tmp_path
|
120
|
+
end
|
121
|
+
|
122
|
+
def upload_file(local_file)
|
123
|
+
path = Pathname.new(local_file)
|
124
|
+
s3_path = bucket_path.join(path.basename)
|
125
|
+
@bucket.put(s3_path, open(path), {}, nil, {
|
126
|
+
'x-amz-storage-class' => 'REDUCED_REDUNDANCY'
|
127
|
+
})
|
128
|
+
end
|
129
|
+
|
130
|
+
def tables
|
131
|
+
unless @tables
|
132
|
+
res = `mysql #{db_connection_options} -e "SHOW TABLES"`
|
133
|
+
# @tables = res.split[1..-1]
|
134
|
+
@tables = res.split[1..10]
|
135
|
+
end
|
136
|
+
@tables
|
137
|
+
end
|
138
|
+
|
139
|
+
def bucket_path
|
140
|
+
@bucket_path ||= Pathname.new("mysql").join(@time.strftime("%Y-%m-%d-%H-%M"))
|
141
|
+
end
|
142
|
+
end # class Dumper
|
143
|
+
|
144
|
+
|
145
|
+
class Loader
|
146
|
+
include Helper
|
147
|
+
include FileUtils
|
148
|
+
|
149
|
+
def initialize(config)
|
150
|
+
@config = config
|
151
|
+
initialize_s3
|
152
|
+
end
|
153
|
+
|
154
|
+
def skip_tables
|
155
|
+
config[:skip_tables] || []
|
156
|
+
end
|
157
|
+
|
158
|
+
def load_latest
|
159
|
+
prefix = backups.first
|
160
|
+
|
161
|
+
tmp_dir = tmp_path.join(prefix.split("/").last)
|
162
|
+
mkdir_p tmp_dir # Creates a directory for day of the backup downloaded
|
163
|
+
|
164
|
+
puts "Downloading backups ..."
|
165
|
+
@bucket.keys(:prefix => prefix).each do |key|
|
166
|
+
filename = File.basename(key.name)
|
167
|
+
|
168
|
+
next if filename.match(/\.csv\.gz$/) && skip_tables.include?(File.basename(filename, ".csv.gz"))
|
169
|
+
|
170
|
+
print " - Downloading #{filename} ... "
|
171
|
+
File.open(tmp_dir.join(filename), "wb") do |f|
|
172
|
+
f.write key.get
|
173
|
+
end
|
174
|
+
print "complete.\n"
|
175
|
+
|
176
|
+
# gunzip file
|
177
|
+
print " -- Inflating #{filename} ... "
|
178
|
+
`gunzip #{tmp_dir.join(filename)}`
|
179
|
+
print "complete.\n"
|
180
|
+
end
|
181
|
+
|
182
|
+
# Load schemas
|
183
|
+
puts "Loading schema."
|
184
|
+
cmd = "cat #{tmp_dir}/*.sql | "
|
185
|
+
cmd += "mysql #{db_connection_options}"
|
186
|
+
puts cmd
|
187
|
+
`#{cmd}`
|
188
|
+
|
189
|
+
# Load data
|
190
|
+
|
191
|
+
puts "Loading data."
|
192
|
+
cmd = "mysqlimport #{db_connection_options}"
|
193
|
+
cmd += csv_options
|
194
|
+
Dir["#{tmp_dir}/*.csv"].each do |table|
|
195
|
+
print " - Importing #{File.basename(table, ".csv")} ... "
|
196
|
+
puts cmd + " " + table
|
197
|
+
`#{cmd} #{table}`
|
198
|
+
print "complete.\n"
|
199
|
+
end
|
200
|
+
|
201
|
+
puts "Backup loaded."
|
202
|
+
ensure
|
203
|
+
rm_r tmp_dir, :force => true
|
204
|
+
end
|
205
|
+
|
206
|
+
# Get a list of backups stored on S3.
|
207
|
+
#
|
208
|
+
# Returns an array of s3 paths that look like:
|
209
|
+
#
|
210
|
+
# mysql/YYYY-MM-DD-HH-MM
|
211
|
+
#
|
212
|
+
# Array elements are sorted with the latest date first.
|
213
|
+
def backups
|
214
|
+
unless @backups
|
215
|
+
@backups = []
|
216
|
+
# Backups are stored in the mysql/ directory
|
217
|
+
@bucket.s3.interface.incrementally_list_bucket(@bucket.name, {
|
218
|
+
:prefix => "mysql/", :delimiter => "/"
|
219
|
+
}) do |item|
|
220
|
+
@backups += item[:common_prefixes]
|
221
|
+
end
|
222
|
+
@backups = @backups.sort { |a,b| b <=> a }
|
223
|
+
end
|
224
|
+
@backups
|
225
|
+
end
|
226
|
+
|
227
|
+
def tables
|
228
|
+
unless @tables
|
229
|
+
res = `mysql #{db_connection_options} -e "SHOW TABLES"`
|
230
|
+
# @tables = res.split[1..-1]
|
231
|
+
@tables = res.split[1..10]
|
232
|
+
end
|
233
|
+
@tables
|
234
|
+
end
|
235
|
+
|
236
|
+
def initialize_directories
|
237
|
+
mkdir_p tmp_dir
|
238
|
+
end
|
239
|
+
end # class Loader
|
240
|
+
end
|
data/mysql_truck.gemspec
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "mysql_truck/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "mysql_truck"
|
7
|
+
s.version = MysqlTruck::VERSION
|
8
|
+
s.authors = ["Peter Bui", "8tracks"]
|
9
|
+
s.email = ["peter@paydrotalks.com"]
|
10
|
+
s.homepage = ""
|
11
|
+
s.summary = %q{Mysql database backup tool. Dumps/Loads to/from S3.}
|
12
|
+
s.description = %q{Mysql database backup tool. Dumps/Loads to/from S3.}
|
13
|
+
|
14
|
+
s.files = `git ls-files`.split("\n")
|
15
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
16
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
17
|
+
s.require_paths = ["lib"]
|
18
|
+
|
19
|
+
s.add_runtime_dependency "right_aws"
|
20
|
+
end
|
metadata
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: mysql_truck
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Peter Bui
|
9
|
+
- 8tracks
|
10
|
+
autorequire:
|
11
|
+
bindir: bin
|
12
|
+
cert_chain: []
|
13
|
+
date: 2011-09-15 00:00:00.000000000 -07:00
|
14
|
+
default_executable:
|
15
|
+
dependencies:
|
16
|
+
- !ruby/object:Gem::Dependency
|
17
|
+
name: right_aws
|
18
|
+
requirement: &2163188460 !ruby/object:Gem::Requirement
|
19
|
+
none: false
|
20
|
+
requirements:
|
21
|
+
- - ! '>='
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: '0'
|
24
|
+
type: :runtime
|
25
|
+
prerelease: false
|
26
|
+
version_requirements: *2163188460
|
27
|
+
description: Mysql database backup tool. Dumps/Loads to/from S3.
|
28
|
+
email:
|
29
|
+
- peter@paydrotalks.com
|
30
|
+
executables:
|
31
|
+
- mysql_truck
|
32
|
+
extensions: []
|
33
|
+
extra_rdoc_files: []
|
34
|
+
files:
|
35
|
+
- .gitignore
|
36
|
+
- Gemfile
|
37
|
+
- Rakefile
|
38
|
+
- bin/mysql_truck
|
39
|
+
- lib/mysql_truck.rb
|
40
|
+
- lib/mysql_truck/version.rb
|
41
|
+
- mysql_truck.gemspec
|
42
|
+
has_rdoc: true
|
43
|
+
homepage: ''
|
44
|
+
licenses: []
|
45
|
+
post_install_message:
|
46
|
+
rdoc_options: []
|
47
|
+
require_paths:
|
48
|
+
- lib
|
49
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
56
|
+
none: false
|
57
|
+
requirements:
|
58
|
+
- - ! '>='
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '0'
|
61
|
+
requirements: []
|
62
|
+
rubyforge_project:
|
63
|
+
rubygems_version: 1.6.2
|
64
|
+
signing_key:
|
65
|
+
specification_version: 3
|
66
|
+
summary: Mysql database backup tool. Dumps/Loads to/from S3.
|
67
|
+
test_files: []
|