mysql_truck 0.5.7.1 → 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/bin/mysql_truck +9 -0
- data/lib/mysql_truck/dumper.rb +63 -28
- data/lib/mysql_truck/helper.rb +1 -0
- data/lib/mysql_truck/loader.rb +33 -4
- data/lib/mysql_truck/version.rb +1 -1
- data/lib/mysql_truck.rb +2 -0
- data/mysql_truck.gemspec +2 -1
- metadata +22 -6
data/.gitignore
CHANGED
data/bin/mysql_truck
CHANGED
@@ -64,11 +64,20 @@ parser = OptionParser.new do |opts|
|
|
64
64
|
options[:dump_dir] = dir
|
65
65
|
end
|
66
66
|
|
67
|
+
opts.on("--grantees GRANTEE", "List of comma separated S3 account ids that can access this data") do |g|
|
68
|
+
options[:grantees] = g
|
69
|
+
end
|
70
|
+
|
67
71
|
opts.on("-t", "--skip-tables TABLES",
|
68
72
|
"List of tables to skip separated by commas.") do |tables|
|
69
73
|
options[:skip_data_for_tables] = tables.split(",")
|
70
74
|
end
|
71
75
|
|
76
|
+
opts.on("-o", "--only-tables TABLES",
|
77
|
+
"List of tables to dump/load.") do |tables|
|
78
|
+
options[:only_tables] = tables.split(",")
|
79
|
+
end
|
80
|
+
|
72
81
|
opts.on("-e", "--exec-smartly",
|
73
82
|
"On dumping, do not dump tables that have already been dumped. On loading, if the files were already downloaded, do not redownload. This option allows for resuming a previous load/dump that failed.") do
|
74
83
|
options[:smartly] = true
|
data/lib/mysql_truck/dumper.rb
CHANGED
@@ -22,11 +22,9 @@ module MysqlTruck
|
|
22
22
|
def dump_data
|
23
23
|
tables.each do |table|
|
24
24
|
puts "Dumping #{table}..."
|
25
|
-
next if
|
25
|
+
next if compressed_files_exist?(table) && smartly?
|
26
26
|
|
27
27
|
if dump_table?(table)
|
28
|
-
|
29
|
-
# This command creates a table_name.sql and a table_name.txt file
|
30
28
|
base_cmd = "mysqldump --quick"
|
31
29
|
|
32
30
|
# Dump schema
|
@@ -37,11 +35,23 @@ module MysqlTruck
|
|
37
35
|
`#{schema_cmd}`
|
38
36
|
|
39
37
|
# Dump data
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
38
|
+
puts "Writing data for #{table} ..."
|
39
|
+
mysql = Mysql.init
|
40
|
+
mysql.options(Mysql::SET_CHARSET_NAME, "utf8")
|
41
|
+
mysql.connect(
|
42
|
+
config[:host],
|
43
|
+
config[:username],
|
44
|
+
config[:password],
|
45
|
+
config[:database],
|
46
|
+
config[:port]
|
47
|
+
)
|
48
|
+
mysql.query("SET NAMES utf8")
|
49
|
+
CSV.open(filename(table)[:data_file], 'wb', :col_sep => "\t") do |csv|
|
50
|
+
result = mysql.query("SELECT * FROM #{table}")
|
51
|
+
result.each do |row|
|
52
|
+
csv << row.collect { |col| col && col.match(/\t/) ? " " : col }
|
53
|
+
end
|
54
|
+
end
|
45
55
|
end
|
46
56
|
|
47
57
|
if split_schema_file?(table)
|
@@ -64,15 +74,17 @@ module MysqlTruck
|
|
64
74
|
end
|
65
75
|
end
|
66
76
|
|
67
|
-
if
|
68
|
-
puts "
|
69
|
-
`gzip #{filename(table)[:no_index_schema_file]}`
|
77
|
+
if compress_files?(table)
|
78
|
+
puts "compressing #{filename(table)[:no_index_schema_file]}."
|
70
79
|
|
71
|
-
|
72
|
-
|
80
|
+
compression_cmd = "lzop -8 -U"
|
81
|
+
`#{compression_cmd} #{filename(table)[:no_index_schema_file]}`
|
73
82
|
|
74
|
-
puts "
|
75
|
-
|
83
|
+
puts "compressing #{filename(table)[:alter_table_file]}."
|
84
|
+
`#{compression_cmd} #{filename(table)[:alter_table_file]}`
|
85
|
+
|
86
|
+
puts "compressing #{filename(table)[:data_file]}."
|
87
|
+
`#{compression_cmd} #{filename(table)[:data_file]}`
|
76
88
|
end
|
77
89
|
|
78
90
|
puts "#{table} dumped.\n\n"
|
@@ -81,7 +93,7 @@ module MysqlTruck
|
|
81
93
|
|
82
94
|
def upload
|
83
95
|
Dir["#{tmp_path}/*"].each do |file|
|
84
|
-
next if File.extname(file) != ".
|
96
|
+
next if File.extname(file) != ".lzo"
|
85
97
|
puts "Uploading #{file} ..."
|
86
98
|
upload_file file
|
87
99
|
end
|
@@ -96,10 +108,31 @@ module MysqlTruck
|
|
96
108
|
|
97
109
|
def upload_file(local_file)
|
98
110
|
path = Pathname.new(local_file)
|
99
|
-
|
111
|
+
|
112
|
+
# Store all schema files in the <DATE>/schema/ dir
|
113
|
+
if local_file.match(/\.indices\.sql/) || local_file.match(/\.no_index\.sql/)
|
114
|
+
s3_path = bucket_path.join("_schema", path.basename)
|
115
|
+
|
116
|
+
# Store data files in <DATE>/<TABLE>/<TABLE>.data.tsv.lzo
|
117
|
+
elsif local_file.match(/\.data\.tsv/)
|
118
|
+
table_name = File.basename(local_file).match(/(.*)\.data\.tsv\.lzo/)[1]
|
119
|
+
s3_path = bucket_path.join(table_name, path.basename)
|
120
|
+
puts "Path: #{s3_path}"
|
121
|
+
|
122
|
+
else
|
123
|
+
puts "** Couldn't upload #{local_file}!!"
|
124
|
+
puts "** Couldn't upload #{local_file}!!"
|
125
|
+
return
|
126
|
+
end
|
127
|
+
puts s3_path
|
100
128
|
@bucket.put(s3_path, open(path), {}, nil, {
|
101
129
|
'x-amz-storage-class' => 'REDUCED_REDUNDANCY'
|
102
130
|
})
|
131
|
+
|
132
|
+
key = RightAws::S3::Key.create(@bucket, s3_path)
|
133
|
+
config[:grantees].split(",").each do |grantee|
|
134
|
+
RightAws::S3::Grantee.new(key, grantee, "FULL_CONTROL", :apply)
|
135
|
+
end
|
103
136
|
end
|
104
137
|
|
105
138
|
def tables
|
@@ -124,19 +157,21 @@ module MysqlTruck
|
|
124
157
|
|
125
158
|
# altered mysql dump files
|
126
159
|
:alter_table_file => tmp_path.join("#{table}.indices.sql"),
|
127
|
-
:data_file => tmp_path.join("#{table}.data.sql"),
|
128
160
|
|
129
|
-
#
|
130
|
-
:
|
131
|
-
|
132
|
-
|
161
|
+
# Tab seperated data
|
162
|
+
:data_file => tmp_path.join("#{table}.data.tsv"),
|
163
|
+
|
164
|
+
# compressed filenames
|
165
|
+
:compressed_no_index_schema_file => tmp_path.join("#{table}.no_index.sql.lzo"),
|
166
|
+
:compressed_alter_table_file => tmp_path.join("#{table}.indices.sql.lzo"),
|
167
|
+
:compressed_data_file => tmp_path.join("#{table}.data.tsv.lzo"),
|
133
168
|
}
|
134
169
|
end
|
135
170
|
|
136
|
-
def
|
137
|
-
filename(table)[:
|
138
|
-
filename(table)[:
|
139
|
-
filename(table)[:
|
171
|
+
def compressed_files_exist?(table)
|
172
|
+
filename(table)[:compressed_no_index_schema_file].file? &&
|
173
|
+
filename(table)[:compressed_alter_table_file].file? &&
|
174
|
+
filename(table)[:compressed_data_file].file?
|
140
175
|
end
|
141
176
|
|
142
177
|
|
@@ -160,14 +195,14 @@ module MysqlTruck
|
|
160
195
|
)
|
161
196
|
end
|
162
197
|
|
163
|
-
def
|
198
|
+
def compress_files?(table)
|
164
199
|
!smartly? ||
|
165
200
|
(
|
166
201
|
smartly? &&
|
167
202
|
filename(table)[:data_file].exist? &&
|
168
203
|
filename(table)[:no_index_schema_file].exist? &&
|
169
204
|
filename(table)[:alter_table_file].exist? &&
|
170
|
-
!
|
205
|
+
!compressed_files_exist?(table)
|
171
206
|
)
|
172
207
|
end
|
173
208
|
end # class Dumper
|
data/lib/mysql_truck/helper.rb
CHANGED
@@ -18,6 +18,7 @@ module MysqlTruck
|
|
18
18
|
opts = %Q[ -u #{config[:username]} ]
|
19
19
|
opts += %Q[ -p"#{config[:password]}" ] unless config[:password].nil?
|
20
20
|
opts += %Q[ -h #{config[:host]} --default-character-set=utf8 ]
|
21
|
+
opts += %Q[ -P #{config[:port]} ] if config[:port]
|
21
22
|
opts += %Q[ #{config[:database]} ]
|
22
23
|
opts
|
23
24
|
end
|
data/lib/mysql_truck/loader.rb
CHANGED
@@ -27,17 +27,23 @@ module MysqlTruck
|
|
27
27
|
@time = Time.new(*backup_date_str.split("-"))
|
28
28
|
initialize_directories
|
29
29
|
|
30
|
-
puts "Download &
|
30
|
+
puts "Download & decompressing backups"
|
31
31
|
puts "-------------------"
|
32
32
|
|
33
33
|
|
34
34
|
@bucket.keys(:prefix => s3_path).each do |key|
|
35
|
+
next if key.to_s.match(/\/$/)
|
35
36
|
next unless (filename = download_file(key))
|
36
37
|
|
37
|
-
# gunzip file
|
38
38
|
if tmp_path.join(filename).exist?
|
39
39
|
print " - Inflating #{filename} ... "
|
40
|
-
|
40
|
+
|
41
|
+
if File.extname(filename).match(".lzo")
|
42
|
+
decompress_cmd = "lzop -d -f -U"
|
43
|
+
else
|
44
|
+
decompress_cmd = "gunzip -f"
|
45
|
+
end
|
46
|
+
time = benchmark { `#{decompress_cmd} #{tmp_path.join(filename)}` }
|
41
47
|
print "complete (#{formatted_time time}).\n"
|
42
48
|
end
|
43
49
|
end
|
@@ -61,6 +67,7 @@ module MysqlTruck
|
|
61
67
|
index_file = tmp_path.join("#{table}.indices.sql")
|
62
68
|
data_file = tmp_path.join("#{table}.data.sql")
|
63
69
|
csv_data_file = tmp_path.join("#{table}.csv")
|
70
|
+
tsv_data_file = tmp_path.join("#{table}.data.tsv")
|
64
71
|
|
65
72
|
|
66
73
|
if schema_file.exist?
|
@@ -88,6 +95,11 @@ module MysqlTruck
|
|
88
95
|
print " - Importing #{File.basename(csv_data_file)} ... "
|
89
96
|
csv_data_file = import_csv_file(table, backup_date_str, csv_data_file)
|
90
97
|
File.delete(csv_data_file)
|
98
|
+
|
99
|
+
elsif tsv_data_file.exist?
|
100
|
+
print " - Importing #{File.basename(tsv_data_file)} ... "
|
101
|
+
tsv_data_file = import_tsv_file(table, backup_date_str, tsv_data_file)
|
102
|
+
File.delete(tsv_data_file)
|
91
103
|
end
|
92
104
|
|
93
105
|
if index_file.exist?
|
@@ -95,7 +107,7 @@ module MysqlTruck
|
|
95
107
|
execute_sql_file(table, backup_date_str, index_file)
|
96
108
|
index_file.delete
|
97
109
|
end
|
98
|
-
|
110
|
+
|
99
111
|
end
|
100
112
|
|
101
113
|
puts "Backup loaded."
|
@@ -138,6 +150,23 @@ module MysqlTruck
|
|
138
150
|
file_path
|
139
151
|
end
|
140
152
|
|
153
|
+
def import_tsv_file(table, backup_date_str, file_path)
|
154
|
+
if config[:date_suffix]
|
155
|
+
old_file_path = file_path
|
156
|
+
file_path = file_path.to_s.gsub("/#{table}.data.", "/#{table}_#{backup_date_str.gsub(/-/, "")}.")
|
157
|
+
# move file
|
158
|
+
`mv #{old_file_path} #{file_path}`
|
159
|
+
end
|
160
|
+
|
161
|
+
time = benchmark do
|
162
|
+
`mysqlimport --local --compress #{db_connection_options} #{file_path}`
|
163
|
+
end
|
164
|
+
|
165
|
+
print "complete (#{formatted_time time}).\n"
|
166
|
+
|
167
|
+
file_path
|
168
|
+
end
|
169
|
+
|
141
170
|
def download_file(key)
|
142
171
|
filename = File.basename(key.name)
|
143
172
|
|
data/lib/mysql_truck/version.rb
CHANGED
data/lib/mysql_truck.rb
CHANGED
data/mysql_truck.gemspec
CHANGED
@@ -16,5 +16,6 @@ Gem::Specification.new do |s|
|
|
16
16
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
17
17
|
s.require_paths = ["lib"]
|
18
18
|
|
19
|
-
s.add_runtime_dependency "right_aws"
|
19
|
+
s.add_runtime_dependency "right_aws", "~> 2.1.0"
|
20
|
+
s.add_runtime_dependency "mysql", "~> 2.8.1"
|
20
21
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mysql_truck
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -11,24 +11,40 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date:
|
14
|
+
date: 2013-01-25 00:00:00.000000000 Z
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: right_aws
|
18
18
|
requirement: !ruby/object:Gem::Requirement
|
19
19
|
none: false
|
20
20
|
requirements:
|
21
|
-
- -
|
21
|
+
- - ~>
|
22
22
|
- !ruby/object:Gem::Version
|
23
|
-
version:
|
23
|
+
version: 2.1.0
|
24
24
|
type: :runtime
|
25
25
|
prerelease: false
|
26
26
|
version_requirements: !ruby/object:Gem::Requirement
|
27
27
|
none: false
|
28
28
|
requirements:
|
29
|
-
- -
|
29
|
+
- - ~>
|
30
30
|
- !ruby/object:Gem::Version
|
31
|
-
version:
|
31
|
+
version: 2.1.0
|
32
|
+
- !ruby/object:Gem::Dependency
|
33
|
+
name: mysql
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
35
|
+
none: false
|
36
|
+
requirements:
|
37
|
+
- - ~>
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: 2.8.1
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
none: false
|
44
|
+
requirements:
|
45
|
+
- - ~>
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 2.8.1
|
32
48
|
description: Mysql database backup tool. Dumps/Loads to/from S3.
|
33
49
|
email:
|
34
50
|
- peter@paydrotalks.com
|