mysql_truck 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,4 @@
1
+ *.gem
2
+ .bundle
3
+ Gemfile.lock
4
+ pkg/*
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in mysql_truck.gemspec
4
+ gemspec
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
data/bin/mysql_truck ADDED
@@ -0,0 +1,90 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'optparse'
4
+ require 'mysql_truck'
5
+ require 'yaml'
6
+
7
+ options = {}
8
+
9
+ parser = OptionParser.new do |opts|
10
+ opts.banner = "Usage: #{File.basename(__FILE__)} (dump|load) [options]"
11
+
12
+ opts.separator ""
13
+ opts.separator "Required Options:"
14
+
15
+ opts.on("-c", "--config FILE", "Configuration yaml file") do |file|
16
+ options[:config] = YAML.load_file(Pathname.new(file).expand_path)
17
+ end
18
+
19
+ opts.separator ""
20
+ opts.separator "Or specify options individually:"
21
+ opts.separator "(options here override options from the config if specified)"
22
+
23
+ opts.on("-H", "--host HOST", "Database host") do |host|
24
+ options[:host] = host
25
+ end
26
+
27
+ opts.on("-u", "--user USERNAME", "Database user") do |user|
28
+ options[:username] = user
29
+ end
30
+
31
+ opts.on("-p", "--password PASSWORD", "Database password") do |password|
32
+ options[:password] = password
33
+ end
34
+
35
+ opts.on("-d", "--database DATABASE_NAME", "Database name") do |db|
36
+ options[:database] = db
37
+ end
38
+
39
+ opts.on("-a", "--access-key KEY", "S3 Access Key") do |key|
40
+ options[:s3_access_key] = key
41
+ end
42
+
43
+ opts.on("-s", "--secret-access-key KEY", "S3 Secret Acess Key") do |key|
44
+ options[:s3_secret_access_key] = key
45
+ end
46
+
47
+ opts.on("-t", "--skip-tables TABLES",
48
+ "List of tables to skip separated by commas.") do |tables|
49
+ options[:skip_tables] = tables.split(",")
50
+ end
51
+
52
+ opts.on_tail("-h", "--help", "Show this message") do
53
+ puts opts
54
+ exit
55
+ end
56
+ end
57
+
58
+ begin
59
+ parser.parse!(ARGV)
60
+
61
+ if options[:config]
62
+ config = {}
63
+ options.delete(:config).each do |k,v|
64
+ config[k.intern] = v
65
+ end
66
+
67
+ options = config.merge(options)
68
+ end
69
+
70
+ missing_opts = []
71
+ [ :s3_access_key, :s3_secret_access_key,
72
+ :host, :username, :database
73
+ ].each do |opt|
74
+ missing_opts << opt unless options.include?(opt)
75
+ end
76
+
77
+ if missing_opts.size > 0
78
+ puts "Missing the following options:"
79
+ missing_opts.each { |o| puts " * #{o}" }
80
+ puts
81
+ puts parser
82
+ exit
83
+ end
84
+ rescue OptionParser::ParseError => e
85
+ puts "\nPlease ensure you have the correct options\n\n"
86
+ puts parser
87
+ exit
88
+ end
89
+
90
+ MysqlTruck.run(ARGV.first, options)
@@ -0,0 +1,240 @@
1
+ require "mysql_truck/version"
2
+ require "right_aws"
3
+ require 'fileutils'
4
+ require 'pathname'
5
+
6
+ # MysqlTruck
7
+ #
8
+ # MySQL backup tool that stores backups in S3.
9
+ #
10
+ # Requires server to have right_aws installed
11
+ module MysqlTruck
12
+ include FileUtils
13
+
14
+ def self.run(action, config)
15
+ case action.intern
16
+ when :dump
17
+ Dumper.new(config).dump
18
+ when :load
19
+ Loader.new(config).load_latest
20
+ else
21
+ puts "Unknown action #{action}"
22
+ end
23
+ end
24
+
25
+ module Helper
26
+ include FileUtils
27
+
28
+ def config
29
+ @config
30
+ end
31
+
32
+ def initialize_s3
33
+ @s3 = RightAws::S3.new(
34
+ config[:s3_access_key],
35
+ config[:s3_secret_access_key])
36
+ @bucket = @s3.bucket("8tr.db_backups")
37
+ end
38
+
39
+ def db_connection_options
40
+ opts = %Q[ -u #{config[:username]} ]
41
+ opts += %Q[ -p"#{config[:password]}" ] unless config[:password].nil?
42
+ opts += %Q[ -h #{config[:host]} --default-character-set=utf8 ]
43
+ opts += %Q[ #{config[:database]} ]
44
+ opts
45
+ end
46
+
47
+ def csv_options
48
+ " --fields-enclosed-by=\\\" --fields-terminated-by=, "
49
+ end
50
+
51
+ def tmp_path
52
+ unless @tmp_path
53
+ @tmp_path = config[:tmp_dir] || Pathname.new("/data/s3backup")
54
+ @tmp_path = @tmp_path.join(@time.to_i.to_s) if @time # Only set with the Dumper class
55
+ end
56
+
57
+ @tmp_path
58
+ end
59
+
60
+ end
61
+
62
+ class Dumper
63
+ include FileUtils
64
+ include Helper
65
+
66
+ def initialize(config)
67
+ @config = config
68
+ @time = Time.now
69
+
70
+ initialize_s3
71
+ initialize_directories
72
+ end
73
+
74
+ def dump
75
+ dump_data
76
+ upload
77
+
78
+ ensure
79
+ # rm_r tmp_path, :force => true
80
+ end
81
+
82
+ def dump_data
83
+ tables.each do |table|
84
+ schema_file = tmp_path.join("#{table}.sql")
85
+ csv_file = tmp_path.join("#{table}.txt")
86
+ puts "Dumping #{table}."
87
+
88
+ # This command creates a table_name.sql and a table_name.txt file
89
+ cmd = "mysqldump --quick -T #{tmp_path} "
90
+ cmd += csv_options
91
+ cmd += "#{db_connection_options} #{table}"
92
+ puts cmd
93
+ `#{cmd}`
94
+
95
+ path, file = csv_file.split
96
+ csv_file = path.join("#{file.basename(".txt")}.csv")
97
+ mv path.join(file), csv_file
98
+
99
+ puts "gziping #{schema_file}."
100
+ `gzip #{schema_file}`
101
+
102
+ puts "gziping #{csv_file}."
103
+ `gzip #{csv_file}`
104
+
105
+ puts "#{table} dumped.\n\n"
106
+ end
107
+ end
108
+
109
+ def upload
110
+ Dir["#{tmp_path}/*"].each do |file|
111
+ upload_file file
112
+ end
113
+ puts "Finished uploading backups."
114
+ end
115
+
116
+ private
117
+
118
+ def initialize_directories
119
+ mkdir_p tmp_path
120
+ end
121
+
122
+ def upload_file(local_file)
123
+ path = Pathname.new(local_file)
124
+ s3_path = bucket_path.join(path.basename)
125
+ @bucket.put(s3_path, open(path), {}, nil, {
126
+ 'x-amz-storage-class' => 'REDUCED_REDUNDANCY'
127
+ })
128
+ end
129
+
130
+ def tables
131
+ unless @tables
132
+ res = `mysql #{db_connection_options} -e "SHOW TABLES"`
133
+ # @tables = res.split[1..-1]
134
+ @tables = res.split[1..10]
135
+ end
136
+ @tables
137
+ end
138
+
139
+ def bucket_path
140
+ @bucket_path ||= Pathname.new("mysql").join(@time.strftime("%Y-%m-%d-%H-%M"))
141
+ end
142
+ end # class Dumper
143
+
144
+
145
+ class Loader
146
+ include Helper
147
+ include FileUtils
148
+
149
+ def initialize(config)
150
+ @config = config
151
+ initialize_s3
152
+ end
153
+
154
+ def skip_tables
155
+ config[:skip_tables] || []
156
+ end
157
+
158
+ def load_latest
159
+ prefix = backups.first
160
+
161
+ tmp_dir = tmp_path.join(prefix.split("/").last)
162
+ mkdir_p tmp_dir # Creates a directory for day of the backup downloaded
163
+
164
+ puts "Downloading backups ..."
165
+ @bucket.keys(:prefix => prefix).each do |key|
166
+ filename = File.basename(key.name)
167
+
168
+ next if filename.match(/\.csv\.gz$/) && skip_tables.include?(File.basename(filename, ".csv.gz"))
169
+
170
+ print " - Downloading #{filename} ... "
171
+ File.open(tmp_dir.join(filename), "wb") do |f|
172
+ f.write key.get
173
+ end
174
+ print "complete.\n"
175
+
176
+ # gunzip file
177
+ print " -- Inflating #{filename} ... "
178
+ `gunzip #{tmp_dir.join(filename)}`
179
+ print "complete.\n"
180
+ end
181
+
182
+ # Load schemas
183
+ puts "Loading schema."
184
+ cmd = "cat #{tmp_dir}/*.sql | "
185
+ cmd += "mysql #{db_connection_options}"
186
+ puts cmd
187
+ `#{cmd}`
188
+
189
+ # Load data
190
+
191
+ puts "Loading data."
192
+ cmd = "mysqlimport #{db_connection_options}"
193
+ cmd += csv_options
194
+ Dir["#{tmp_dir}/*.csv"].each do |table|
195
+ print " - Importing #{File.basename(table, ".csv")} ... "
196
+ puts cmd + " " + table
197
+ `#{cmd} #{table}`
198
+ print "complete.\n"
199
+ end
200
+
201
+ puts "Backup loaded."
202
+ ensure
203
+ rm_r tmp_dir, :force => true
204
+ end
205
+
206
+ # Get a list of backups stored on S3.
207
+ #
208
+ # Returns an array of s3 paths that look like:
209
+ #
210
+ # mysql/YYYY-MM-DD-HH-MM
211
+ #
212
+ # Array elements are sorted with the latest date first.
213
+ def backups
214
+ unless @backups
215
+ @backups = []
216
+ # Backups are stored in the mysql/ directory
217
+ @bucket.s3.interface.incrementally_list_bucket(@bucket.name, {
218
+ :prefix => "mysql/", :delimiter => "/"
219
+ }) do |item|
220
+ @backups += item[:common_prefixes]
221
+ end
222
+ @backups = @backups.sort { |a,b| b <=> a }
223
+ end
224
+ @backups
225
+ end
226
+
227
+ def tables
228
+ unless @tables
229
+ res = `mysql #{db_connection_options} -e "SHOW TABLES"`
230
+ # @tables = res.split[1..-1]
231
+ @tables = res.split[1..10]
232
+ end
233
+ @tables
234
+ end
235
+
236
+ def initialize_directories
237
+ mkdir_p tmp_dir
238
+ end
239
+ end # class Loader
240
+ end
@@ -0,0 +1,3 @@
1
+ module MysqlTruck
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,20 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require "mysql_truck/version"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "mysql_truck"
7
+ s.version = MysqlTruck::VERSION
8
+ s.authors = ["Peter Bui", "8tracks"]
9
+ s.email = ["peter@paydrotalks.com"]
10
+ s.homepage = ""
11
+ s.summary = %q{Mysql database backup tool. Dumps/Loads to/from S3.}
12
+ s.description = %q{Mysql database backup tool. Dumps/Loads to/from S3.}
13
+
14
+ s.files = `git ls-files`.split("\n")
15
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
+ s.require_paths = ["lib"]
18
+
19
+ s.add_runtime_dependency "right_aws"
20
+ end
metadata ADDED
@@ -0,0 +1,67 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: mysql_truck
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Peter Bui
9
+ - 8tracks
10
+ autorequire:
11
+ bindir: bin
12
+ cert_chain: []
13
+ date: 2011-09-15 00:00:00.000000000 -07:00
14
+ default_executable:
15
+ dependencies:
16
+ - !ruby/object:Gem::Dependency
17
+ name: right_aws
18
+ requirement: &2163188460 !ruby/object:Gem::Requirement
19
+ none: false
20
+ requirements:
21
+ - - ! '>='
22
+ - !ruby/object:Gem::Version
23
+ version: '0'
24
+ type: :runtime
25
+ prerelease: false
26
+ version_requirements: *2163188460
27
+ description: Mysql database backup tool. Dumps/Loads to/from S3.
28
+ email:
29
+ - peter@paydrotalks.com
30
+ executables:
31
+ - mysql_truck
32
+ extensions: []
33
+ extra_rdoc_files: []
34
+ files:
35
+ - .gitignore
36
+ - Gemfile
37
+ - Rakefile
38
+ - bin/mysql_truck
39
+ - lib/mysql_truck.rb
40
+ - lib/mysql_truck/version.rb
41
+ - mysql_truck.gemspec
42
+ has_rdoc: true
43
+ homepage: ''
44
+ licenses: []
45
+ post_install_message:
46
+ rdoc_options: []
47
+ require_paths:
48
+ - lib
49
+ required_ruby_version: !ruby/object:Gem::Requirement
50
+ none: false
51
+ requirements:
52
+ - - ! '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ required_rubygems_version: !ruby/object:Gem::Requirement
56
+ none: false
57
+ requirements:
58
+ - - ! '>='
59
+ - !ruby/object:Gem::Version
60
+ version: '0'
61
+ requirements: []
62
+ rubyforge_project:
63
+ rubygems_version: 1.6.2
64
+ signing_key:
65
+ specification_version: 3
66
+ summary: Mysql database backup tool. Dumps/Loads to/from S3.
67
+ test_files: []