sqlup 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +5 -0
- data/Manifest.txt +38 -0
- data/README.txt +122 -0
- data/Rakefile +18 -0
- data/bin/sqlup +107 -0
- data/bin/sqlup_control +25 -0
- data/config/environment.rb +0 -0
- data/lib/mysql_backup/entity/files/innodb.rb +77 -0
- data/lib/mysql_backup/entity/files/myisam.rb +62 -0
- data/lib/mysql_backup/entity/files.rb +95 -0
- data/lib/mysql_backup/entity/identifier.rb +179 -0
- data/lib/mysql_backup/entity/logs.rb +44 -0
- data/lib/mysql_backup/entity/mysqldump.rb +57 -0
- data/lib/mysql_backup/entity.rb +38 -0
- data/lib/mysql_backup/librarian/backup.rb +134 -0
- data/lib/mysql_backup/librarian/backup_collection.rb +63 -0
- data/lib/mysql_backup/librarian.rb +176 -0
- data/lib/mysql_backup/server.rb +237 -0
- data/lib/mysql_backup/storage/s3.rb +110 -0
- data/lib/mysql_backup/storage.rb +13 -0
- data/lib/mysql_backup/utilities/factory_create_method.rb +51 -0
- data/lib/mysql_backup.rb +8 -0
- data/lib/sqlup.rb +2 -0
- data/test/unit/mysql_backup/entity/files/files_test.rb +45 -0
- data/test/unit/mysql_backup/entity/files/innodb_test.rb +50 -0
- data/test/unit/mysql_backup/entity/files/myisam_test.rb +42 -0
- data/test/unit/mysql_backup/entity/identifier_test.rb +51 -0
- data/test/unit/mysql_backup/entity/logs_test.rb +13 -0
- data/test/unit/mysql_backup/entity/mysqldump_test.rb +64 -0
- data/test/unit/mysql_backup/librarian/backup_collection_test.rb +52 -0
- data/test/unit/mysql_backup/librarian/backup_test.rb +25 -0
- data/test/unit/mysql_backup/librarian/librarian_test.rb +103 -0
- data/test/unit/mysql_backup/server_test.rb +63 -0
- data/test/unit/mysql_backup/storage/s3_test.rb +69 -0
- data/test/unit/mysql_backup/storage/test_helper.rb +1 -0
- data/test/unit/mysql_backup/test_helper.rb +1 -0
- data/test/unit/mysql_backup/utilities/test_helper.rb +1 -0
- data/test/unit/test_helper.rb +10 -0
- metadata +116 -0
data/History.txt
ADDED
data/Manifest.txt
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
History.txt
|
2
|
+
Manifest.txt
|
3
|
+
README.txt
|
4
|
+
Rakefile
|
5
|
+
bin/sqlup
|
6
|
+
bin/sqlup_control
|
7
|
+
test/unit/mysql_backup/librarian/librarian_test.rb
|
8
|
+
test/unit/mysql_backup/librarian/backup_test.rb
|
9
|
+
test/unit/mysql_backup/librarian/backup_collection_test.rb
|
10
|
+
test/unit/mysql_backup/utilities/test_helper.rb
|
11
|
+
test/unit/mysql_backup/storage/test_helper.rb
|
12
|
+
test/unit/mysql_backup/storage/s3_test.rb
|
13
|
+
test/unit/mysql_backup/entity/files/myisam_test.rb
|
14
|
+
test/unit/mysql_backup/entity/files/files_test.rb
|
15
|
+
test/unit/mysql_backup/entity/files/innodb_test.rb
|
16
|
+
test/unit/mysql_backup/entity/logs_test.rb
|
17
|
+
test/unit/mysql_backup/entity/mysqldump_test.rb
|
18
|
+
test/unit/mysql_backup/entity/identifier_test.rb
|
19
|
+
test/unit/mysql_backup/test_helper.rb
|
20
|
+
test/unit/mysql_backup/server_test.rb
|
21
|
+
test/unit/test_helper.rb
|
22
|
+
lib/mysql_backup/librarian/backup.rb
|
23
|
+
lib/mysql_backup/librarian/backup_collection.rb
|
24
|
+
lib/mysql_backup/entity/files/myisam.rb
|
25
|
+
lib/mysql_backup/entity/files/innodb.rb
|
26
|
+
lib/mysql_backup/entity/identifier.rb
|
27
|
+
lib/mysql_backup/entity/logs.rb
|
28
|
+
lib/mysql_backup/entity/mysqldump.rb
|
29
|
+
lib/mysql_backup/entity/files.rb
|
30
|
+
lib/mysql_backup/storage/s3.rb
|
31
|
+
lib/mysql_backup/utilities/factory_create_method.rb
|
32
|
+
lib/mysql_backup/server.rb
|
33
|
+
lib/mysql_backup/librarian.rb
|
34
|
+
lib/mysql_backup/storage.rb
|
35
|
+
lib/mysql_backup/entity.rb
|
36
|
+
lib/mysql_backup.rb
|
37
|
+
lib/sqlup.rb
|
38
|
+
config/environment.rb
|
data/README.txt
ADDED
@@ -0,0 +1,122 @@
|
|
1
|
+
sqldump
|
2
|
+
by James Moore
|
3
|
+
|
4
|
+
== DESCRIPTION:
|
5
|
+
|
6
|
+
sqlup is a set of libraries and utilities to automate backups of a MySQL server running on Amazon's EC2
|
7
|
+
service to Amazon's S3 storage service.
|
8
|
+
|
9
|
+
=== Quick start
|
10
|
+
|
11
|
+
Create an S3 bucket named 'sqlup' for your backups. (You'll need to choose a different name for your bucket.)
|
12
|
+
|
13
|
+
Install the gem:
|
14
|
+
gem install sqlup
|
15
|
+
|
16
|
+
Put your S3 keys in a .sqluprc file in the backup user's home directory:
|
17
|
+
cat ~/.sqluprc
|
18
|
+
access_key_id: xxxxxxxxxxxxxxxx
|
19
|
+
secret_access_key: xxxxxxxxxxxxxxxxx
|
20
|
+
|
21
|
+
Backup your database:
|
22
|
+
sqlup binary -bucket sqlup
|
23
|
+
|
24
|
+
See what was written:
|
25
|
+
sqlup ls -bucket sqlup
|
26
|
+
|
27
|
+
Start the backup daemon that will store the binary logs as they're written:
|
28
|
+
sqlup_control start -- -logs_delay 10 log_daemon -bucket sqlup
|
29
|
+
|
30
|
+
Retrieve the backup files (where full:type_mysqldump:log_file_domU-12-31-35-00-35-42-bin.000019:log_position_0000000169 is the name of the full backup you want):
|
31
|
+
sqlup get_logs -bucket sqlup -d /tmp
|
32
|
+
sqlup get -bucket sqlup -d /tmp -name full:type_mysqldump:log_file_domU-12-31-35-00-35-42-bin.000019:log_position_0000000169
|
33
|
+
|
34
|
+
=== Usage
|
35
|
+
|
36
|
+
Get help:
|
37
|
+
sqlup �h
|
38
|
+
Back up the data files:
|
39
|
+
sqlup binary -bucket sqlup
|
40
|
+
Back up a mysqldump run:
|
41
|
+
sqlup mysqldump -bucket sqlup
|
42
|
+
Start the sqlup daemon to take a backup every 10 seconds, to the bucket 'sqlup', with a pidfile in /tmp:
|
43
|
+
export SQLUP_PID_DIR=/tmp
|
44
|
+
sqlup_control start -- -logs_delay 10 log_daemon -bucket sqlup
|
45
|
+
Get a list of the backup files:
|
46
|
+
sqlup ls -bucket sqlup
|
47
|
+
Remove a backup file from the bucket 'sqlup':
|
48
|
+
sqlup -bucket sqlup rm -name log:type_complete:log_file_fnord.000002
|
49
|
+
Remove obsolete current logs:
|
50
|
+
obsolete_files=`bin/sqlup -bucket sqlup ls -backup_type log_current -skip_most_recent 3`
|
51
|
+
for i in $obsolete_files ; do
|
52
|
+
sqlup -bucket sqlup rm -name $i
|
53
|
+
done
|
54
|
+
|
55
|
+
You need to specify your access codes using the AWS::S3 environment variables:
|
56
|
+
|
57
|
+
export AMAZON_ACCESS_KEY_ID='xxxxxxx'
|
58
|
+
export AMAZON_SECRET_ACCESS_KEY='xxxxxxxxxx'
|
59
|
+
|
60
|
+
|
61
|
+
It�s primarily targeted at MySQL servers running on Amazon�s EC2 virtual server system,
|
62
|
+
with backups sent to Amazon�s S3 storage service.
|
63
|
+
|
64
|
+
=== What it backs up
|
65
|
+
There are three parts to a MySQL backup system:
|
66
|
+
|
67
|
+
1. The actual MySQL data files (by default, the files in /var/lib/mysql).
|
68
|
+
2. Full dumps using the mysqldump tool.
|
69
|
+
3. The binary log files.
|
70
|
+
|
71
|
+
Normally, you�d make a full copy of your system using either #1 or #2, and then have sqlup make backups of the binary logs to S3 every N seconds.
|
72
|
+
Backups are tarred, gzip�ed, and split to fit into S3 buckets.
|
73
|
+
|
74
|
+
=== FAQs
|
75
|
+
|
76
|
+
* Why would I want to make copies of the data files instead of using mysqldump?
|
77
|
+
|
78
|
+
Speed. Recovering mysqldump files can take a while; recovering when you have copies of the data files is much faster. If you�ve got a small database, though, just use mysqldump. You can test this yourself; just do a mysqldump of your current database, and run it against a MySQL server on another machine. If that�s fast enough for you, you don�t need to worry about the binary files.
|
79
|
+
|
80
|
+
* Can I use the database while it�s being backed up?
|
81
|
+
|
82
|
+
Yes, sort of. The binary backup is going to lock every table in the system until it�s finished making a tarball of all the data files. (A future enhancement will be to use a versioning file system.) As long as no one writes to the database, reads will continue. However, as soon as someone wants to write to a table, it�s very likely that all future readers will be blocked until the backup is finished writing temporary files to disk.
|
83
|
+
|
84
|
+
* Is there an automated recovery system?
|
85
|
+
|
86
|
+
Not yet. There�s a command to get the backups back from S3, but you need to go through the standard mysql recovery process by hand once you have the files.
|
87
|
+
|
88
|
+
|
89
|
+
== FEATURES/PROBLEMS:
|
90
|
+
|
91
|
+
== TODO:
|
92
|
+
|
93
|
+
1. Improve the recovery process.
|
94
|
+
2. Improve the ability to do backups from slaves.
|
95
|
+
|
96
|
+
== REQUIREMENTS:
|
97
|
+
|
98
|
+
* You must run MySQL with binary logging enabled. No logs == no backups.
|
99
|
+
|
100
|
+
== INSTALL:
|
101
|
+
|
102
|
+
gem install sqlup
|
103
|
+
|
104
|
+
== LICENSE:
|
105
|
+
|
106
|
+
sqlup - a backup tool for MySQL, EC2, and S3
|
107
|
+
|
108
|
+
Copyright (C) 2007 James Moore
|
109
|
+
|
110
|
+
This program is free software; you can redistribute it and/or
|
111
|
+
modify it under the terms of the GNU General Public License
|
112
|
+
as published by the Free Software Foundation; either version 2
|
113
|
+
of the License, or (at your option) any later version.
|
114
|
+
|
115
|
+
This program is distributed in the hope that it will be useful,
|
116
|
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
117
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
118
|
+
GNU General Public License for more details.
|
119
|
+
|
120
|
+
You should have received a copy of the GNU General Public License
|
121
|
+
along with this program; if not, write to the Free Software
|
122
|
+
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
data/Rakefile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'hoe'
|
3
|
+
require './lib/mysql_backup'
|
4
|
+
|
5
|
+
Hoe.new('sqlup', MysqlBackup::VERSION) do |p|
|
6
|
+
p.rubyforge_name = 'sqlup'
|
7
|
+
p.author = 'James Moore'
|
8
|
+
p.email = 'banshee@restphone.com'
|
9
|
+
p.summary = "A backup tool for saving MySQL data to Amazon's S3 service"
|
10
|
+
p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
|
11
|
+
p.extra_deps << ['named_arguments', '>= 0.0.5']
|
12
|
+
p.extra_deps << ['optiflag', '>= 0.6.5']
|
13
|
+
end
|
14
|
+
|
15
|
+
Rake::TestTask.new do |t|
|
16
|
+
t.test_files = FileList['test/**/*_test.rb']
|
17
|
+
t.verbose = true
|
18
|
+
end
|
data/bin/sqlup
ADDED
@@ -0,0 +1,107 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'optiflag'
|
5
|
+
require 'pp'
|
6
|
+
require 'logger'
|
7
|
+
require File.dirname(__FILE__) + '/../lib/mysql_backup'
|
8
|
+
|
9
|
+
module Example extend OptiFlagSet
|
10
|
+
keyword 'ls', :description => "Print a list of all backups."
|
11
|
+
keyword 'mysqldump', :description => 'Backup using mysqldump'
|
12
|
+
keyword 'binary', :description => 'Backup mysql data files'
|
13
|
+
keyword 'logs', :description => 'Backup the binary log files'
|
14
|
+
keyword 'log_daemon', :description => "Backup the binary logs every --logs_delay seconds."
|
15
|
+
keyword 'get', :description => 'Get a binary or a mysqldump backup. Requires --name'
|
16
|
+
keyword 'get_logs', :description => "Get all the current and complete logs"
|
17
|
+
keyword 'rm', :description => 'Remove a backup (use -name to specify which backup to remove)'
|
18
|
+
flag 'bucket', :description => "The name of the S3 bucket"
|
19
|
+
optional_flag 'backup_type', :description => "(ls) The kind of backup"
|
20
|
+
optional_flag 'skip_most_recent', :description => "(ls) Do not display the most recent N logs. Usually used to get a list of obsolete current logs."
|
21
|
+
optional_flag 'name', :description => '(get, rm) The name of the backup to process'
|
22
|
+
optional_flag "engine", :description => "The storage engine. The default (and currently the only option) is s3."
|
23
|
+
optional_flag "get_directory", :alternate_forms => 'd', :description => "The directory to write files for the get command (defaults to the current directory)"
|
24
|
+
optional_flag 'logs_delay', :description => 'The delay (in seconds, as a float) to use for log_daemon. Defaults to 1.'
|
25
|
+
optional_switch_flag 'verbose', :alternate_forms => %w(v) do
|
26
|
+
description 'Send verbose output'
|
27
|
+
end
|
28
|
+
and_process!
|
29
|
+
end
|
30
|
+
|
31
|
+
flags = ARGV.flags
|
32
|
+
|
33
|
+
if flags.get && !flags.name
|
34
|
+
raise RuntimeError, "You must specify the name of a backup (--name)"
|
35
|
+
end
|
36
|
+
|
37
|
+
if flags.verbose?
|
38
|
+
log = Logger.new $stderr
|
39
|
+
log.level = Logger::DEBUG
|
40
|
+
else
|
41
|
+
log = nil
|
42
|
+
end
|
43
|
+
|
44
|
+
engine = flags.engine || 's3'
|
45
|
+
|
46
|
+
case engine
|
47
|
+
when 's3'
|
48
|
+
# Read the keys from the $HOME/.sqluprc file
|
49
|
+
args = {:log => log, :bucket => flags.bucket}
|
50
|
+
p = Pathname.new(ENV['HOME']) + '.sqluprc'
|
51
|
+
if p.readable?
|
52
|
+
keys = YAML::load p.open
|
53
|
+
keys.each_pair do |k,v|
|
54
|
+
args[k.to_sym] = v
|
55
|
+
end
|
56
|
+
else
|
57
|
+
raise RuntimeError, ("Cannot open #{p.to_s}; that file should have two lines, one for access_key_id: yourkey and the other for secret_access_key: yourotherkey")
|
58
|
+
end
|
59
|
+
engine = MysqlBackup::Storage::S3.new args
|
60
|
+
engine.read_existing_objects
|
61
|
+
end
|
62
|
+
librarian = MysqlBackup::Librarian.new :log => log, :storage => engine, :user => 'root', :bucket => 'banshee', :sock => '/var/lib/mysql/mysql.sock'
|
63
|
+
|
64
|
+
begin
|
65
|
+
if flags.ls?
|
66
|
+
klass = Kernel.const_get flags.backup_type rescue nil
|
67
|
+
klass ||= case flags.backup_type
|
68
|
+
when /log_complete/: MysqlBackup::Librarian::Backup::Log::Complete
|
69
|
+
when /log_current/: MysqlBackup::Librarian::Backup::Log::Current
|
70
|
+
when /log/: MysqlBackup::Librarian::Backup::Log
|
71
|
+
end
|
72
|
+
klass ||= Object
|
73
|
+
range = 0..-1
|
74
|
+
if flags.skip_most_recent
|
75
|
+
range = 0..(-(flags.skip_most_recent.to_i) - 1)
|
76
|
+
end
|
77
|
+
puts librarian.ls(klass).slice(range).join("\n")
|
78
|
+
end
|
79
|
+
if flags.mysqldump?
|
80
|
+
librarian.backup_mysqldump
|
81
|
+
end
|
82
|
+
if flags.binary?
|
83
|
+
librarian.backup_data_files
|
84
|
+
end
|
85
|
+
if flags.rm?
|
86
|
+
librarian.rm flags.name
|
87
|
+
end
|
88
|
+
if flags.logs?
|
89
|
+
librarian.backup_binary_logs
|
90
|
+
end
|
91
|
+
if flags.get?
|
92
|
+
librarian.get flags.name, flags.get_directory || '.'
|
93
|
+
end
|
94
|
+
if flags.get_logs?
|
95
|
+
librarian.get_logs flags.get_directory || '.'
|
96
|
+
end
|
97
|
+
if flags.log_daemon?
|
98
|
+
while true
|
99
|
+
librarian.backup_binary_logs
|
100
|
+
delay = flags.logs_delay.to_f
|
101
|
+
sleep(delay == 0 ? 1 : delay)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
rescue Exception => e
|
105
|
+
puts "Exception: #{e.class}: #{e.message}\n\t#{e.backtrace.join("\n\t")}"
|
106
|
+
exit 1
|
107
|
+
end
|
data/bin/sqlup_control
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# Normally run like so:
|
4
|
+
#
|
5
|
+
# export SQLUP_PID_DIR=/tmp
|
6
|
+
# sqlup_control start -- -logs_delay 10 log_daemon
|
7
|
+
#
|
8
|
+
# This will back up your log files every 10 seconds.
|
9
|
+
|
10
|
+
require 'rubygems'
|
11
|
+
require 'daemons'
|
12
|
+
require 'optiflag'
|
13
|
+
|
14
|
+
options = {}
|
15
|
+
|
16
|
+
pid_dir = ENV['SQLUP_PID_DIR']
|
17
|
+
|
18
|
+
if pid_dir
|
19
|
+
options[:dir] = pid_dir
|
20
|
+
options[:dir_mode] = :normal
|
21
|
+
else
|
22
|
+
options[:dir_mode] = :script
|
23
|
+
end
|
24
|
+
|
25
|
+
Daemons.run(File.dirname(__FILE__) + '/sqlup', options)
|
File without changes
|
@@ -0,0 +1,77 @@
|
|
1
|
+
require 'mysql_backup/entity/files'
|
2
|
+
|
3
|
+
# Used to save all of the files for a MySQL Innodb database.
|
4
|
+
#
|
5
|
+
# Normal use is:
|
6
|
+
#
|
7
|
+
# i = Mysql::InnodbFiles.new
|
8
|
+
# array_of_pathname_objects = i.tar_files
|
9
|
+
#
|
10
|
+
# The caller is responsible for removing the files returned
|
11
|
+
# from tar_files.
|
12
|
+
#
|
13
|
+
# See new for a description of the arguments used to create a Mysql::InnodbFiles object
|
14
|
+
# matching your mysql layout.
|
15
|
+
#
|
16
|
+
# You can create instances of Mysql::InnodbFiles by hand, but normally
|
17
|
+
# you'd use MysqlBackup::Server to create them for you.
|
18
|
+
# MysqlBackup::Server
|
19
|
+
# objects will detect where your data files are and fill in the correct options for
|
20
|
+
# Mysql::InnodbFiles.new.
|
21
|
+
#
|
22
|
+
# = What is backed up
|
23
|
+
#
|
24
|
+
# - Innodb data files. Your innodb files must all start with the same string
|
25
|
+
# and end with a sequence of digits. +ibdata001+, +ibdata002+, etc. is fine,
|
26
|
+
# but +firstfile+, +secondfile+ is not supported.
|
27
|
+
# - MyISAM files.
|
28
|
+
# - Log files. You need to specify the directory where the log files are stored;
|
29
|
+
# there's no way to extract this from a running server.
|
30
|
+
# - +mysqladmindump+ files.
|
31
|
+
#
|
32
|
+
# For all of these, we track the log positions (the log file/log position pair).
|
33
|
+
class MysqlBackup::Entity::Files::Innodb < MysqlBackup::Entity::Files
|
34
|
+
attr_accessor :datadir, :innodb_data_home_dir, :innodb_data_file_path
|
35
|
+
|
36
|
+
# Takes the following arguments:
|
37
|
+
#
|
38
|
+
# :datadir => The MySQL data directory.
|
39
|
+
# :innodb_data_home_dir => The directory for innodb files.
|
40
|
+
# :innodb_data_file_path => The root name for the innodb files.
|
41
|
+
# :log_bin_dir => The directory containing the log files.
|
42
|
+
# :log_bin => The prefix of the log files.
|
43
|
+
#
|
44
|
+
# The following files will be backed up with the default base_dir and ib_basename:
|
45
|
+
#
|
46
|
+
# /var/lib/mysql/ibdata*
|
47
|
+
# /var/lib/mysql/*/*.frm - all files in any directory that contain one or more *.frm files
|
48
|
+
# /var/lib/mysql/*/*.MYD - all files in any directory that contain one or more *.MYD files
|
49
|
+
def initialize args = {}
|
50
|
+
set_path_vars %w(datadir innodb_data_home_dir), args
|
51
|
+
end
|
52
|
+
|
53
|
+
# Returns a list of Pathname objects that need to be
|
54
|
+
# backed up for a mysql server using innodb.
|
55
|
+
def required_paths
|
56
|
+
result = []
|
57
|
+
result.concat innodb_files
|
58
|
+
result.concat innodb_database_dirs
|
59
|
+
result.uniq
|
60
|
+
end
|
61
|
+
|
62
|
+
# Returns an array of Pathnames for all databases
|
63
|
+
# in the base directory.
|
64
|
+
#
|
65
|
+
# A database in this case is a directory containing
|
66
|
+
# any files matching *.frm.
|
67
|
+
def innodb_database_dirs
|
68
|
+
result = Pathname.glob(@innodb_data_home_dir_path + '*/*.frm')
|
69
|
+
result = result.map {|d| d.dirname}
|
70
|
+
result.uniq
|
71
|
+
end
|
72
|
+
|
73
|
+
def innodb_files
|
74
|
+
ib = @innodb_data_home_dir_path + @innodb_data_file_path
|
75
|
+
Pathname.glob ib.cleanpath.to_s + '*'
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'mysql_backup/entity/files'
|
2
|
+
|
3
|
+
# Used to save all of the files for a MySQL MyISAM database.
|
4
|
+
#
|
5
|
+
# Normal use is:
|
6
|
+
#
|
7
|
+
# i = Mysql::InnodbFiles.new
|
8
|
+
# array_of_pathname_objects = i.tar_files
|
9
|
+
#
|
10
|
+
# The caller is responsible for removing the files returned
|
11
|
+
# from tar_files.
|
12
|
+
#
|
13
|
+
# See new for a description of the arguments used to create a Mysql::InnodbFiles object
|
14
|
+
# matching your mysql layout.
|
15
|
+
#
|
16
|
+
# You can create instances of Mysql::InnodbFiles by hand, but normally
|
17
|
+
# you'd use MysqlBackup::Server to create them for you.
|
18
|
+
# MysqlBackup::Server
|
19
|
+
# objects will detect where your data files are and fill in the correct options for
|
20
|
+
# Mysql::InnodbFiles.new.
|
21
|
+
#
|
22
|
+
# = What is backed up
|
23
|
+
#
|
24
|
+
# - MyISAM files.
|
25
|
+
#
|
26
|
+
# For all of these, we track the log positions (the log file/log position pair).
|
27
|
+
class MysqlBackup::Entity::Files::Myisam < MysqlBackup::Entity::Files
|
28
|
+
# Takes the following arguments:
|
29
|
+
#
|
30
|
+
# :datadir => The MySQL data directory.
|
31
|
+
#
|
32
|
+
# The following files will be backed up with the default base_dir and ib_basename:
|
33
|
+
#
|
34
|
+
# /var/lib/mysql/*/*.MYD - all files in any directory that contain one or more *.MYD files
|
35
|
+
|
36
|
+
attr_accessor :datadir
|
37
|
+
|
38
|
+
def initialize args = {}
|
39
|
+
super
|
40
|
+
set_path_vars %w(datadir), args
|
41
|
+
end
|
42
|
+
|
43
|
+
# Returns a list of Pathname objects that need to be
|
44
|
+
# backed up for a mysql server using innodb.
|
45
|
+
def required_paths
|
46
|
+
result = []
|
47
|
+
result.concat myisam_database_dirs
|
48
|
+
result.uniq
|
49
|
+
end
|
50
|
+
|
51
|
+
# Returns an array of Pathnames for all databases
|
52
|
+
# in the base directory.
|
53
|
+
#
|
54
|
+
# A database in this case is a directory containing
|
55
|
+
# any files matching *.frm.
|
56
|
+
def myisam_database_dirs
|
57
|
+
result = Pathname.glob(@datadir_path + '*/*.MYD')
|
58
|
+
result.concat Pathname.glob(@datadir_path + '*/*.MYI')
|
59
|
+
result = result.map {|d| d.dirname}
|
60
|
+
result.uniq
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
require 'pathname'
|
2
|
+
require 'tempfile'
|
3
|
+
|
4
|
+
require 'mysql_backup/entity'
|
5
|
+
require 'mysql_backup/entity/identifier'
|
6
|
+
|
7
|
+
# See MysqlBackup::Entity::Files::Innodb and MysqlBackup::Entity::Files::Myisam
|
8
|
+
# for implementations of this abstract base class.
|
9
|
+
class MysqlBackup::Entity::Files < MysqlBackup::Entity
|
10
|
+
def initialize args = {}
|
11
|
+
super()
|
12
|
+
end
|
13
|
+
|
14
|
+
# Create the tar files to back up a mysql instance.
|
15
|
+
#
|
16
|
+
# Takes the following:
|
17
|
+
#
|
18
|
+
# :mysql_server => a MysqlBackup::Server object
|
19
|
+
# :mysql_files => an array of MysqlBackup::Entity::Files objects
|
20
|
+
#
|
21
|
+
# Yields a hash:
|
22
|
+
#
|
23
|
+
# :identifier => a MysqlBackup::Entity::Identifier object
|
24
|
+
# :file => a Pathname object
|
25
|
+
def self.create_tar_files args = {}, &block
|
26
|
+
begin
|
27
|
+
log_data = build_files args
|
28
|
+
log_data[:files].each do |p|
|
29
|
+
process_file(log_data.merge(:file => p), &block)
|
30
|
+
end
|
31
|
+
ensure
|
32
|
+
# files.each {|f| f.unlink if f.exist?}
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
# Create the actual files in a
|
37
|
+
# <tt>with_lock</tt> block.
|
38
|
+
def self.build_files args #:nodoc:
|
39
|
+
log_data = nil
|
40
|
+
args[:mysql_server].with_lock do |l|
|
41
|
+
log_data = l
|
42
|
+
mysql_file_objs = args[:mysql_files]
|
43
|
+
mysql_file_objs.each do |o|
|
44
|
+
o.confirm_required_paths_are_readable
|
45
|
+
end
|
46
|
+
path_strings = (mysql_file_objs.map {|o| o.required_path_strings}).flatten.uniq
|
47
|
+
log_data[:files] = tar_files path_strings
|
48
|
+
log_data[:n_parts] = log_data[:files].length
|
49
|
+
end
|
50
|
+
log_data
|
51
|
+
end
|
52
|
+
|
53
|
+
# Process each file with the specified block
|
54
|
+
def self.process_file args, &block #:nodoc:
|
55
|
+
part_number = args[:file].to_s[/.*(\d+)$/, 1].to_i
|
56
|
+
identifier = MysqlBackup::Entity::Identifier.create_object args.merge(:category => :full, :type => :binary, :part_number => part_number)
|
57
|
+
block.call(:identifier => identifier, :file => args[:file])
|
58
|
+
end
|
59
|
+
|
60
|
+
def confirm_required_paths_are_readable
|
61
|
+
required_paths.each do |p|
|
62
|
+
raise RuntimeError, "Not readable: #{p}" unless p.readable?
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
# Returns a list of path strings that need to be
|
67
|
+
# backed up.
|
68
|
+
def required_path_strings
|
69
|
+
files = required_paths.map {|p| p.cleanpath.to_s}
|
70
|
+
end
|
71
|
+
|
72
|
+
def self.do_tar args #:nodoc:
|
73
|
+
log = args[:log]
|
74
|
+
log && log.info("running #{cmd}")
|
75
|
+
system args[:cmd] or raise RuntimeError, "The command failed with status #{$?}"
|
76
|
+
end
|
77
|
+
|
78
|
+
# Given a set of variable names, create a
|
79
|
+
# set of matching Pathname objects with
|
80
|
+
# <tt>_path</tt> appended to the name.
|
81
|
+
def set_path_vars var_names, args = {} #:nodoc:
|
82
|
+
args.each_pair do |k,v|
|
83
|
+
send "#{k}=", v
|
84
|
+
end
|
85
|
+
var_names.each do |p|
|
86
|
+
instance_var_name = "@#{p}"
|
87
|
+
instance_var_value = instance_variable_get(instance_var_name)
|
88
|
+
path_instance_var_name = "@#{p}_path"
|
89
|
+
raise RuntimeError, "Must pass :#{p}" unless instance_var_value
|
90
|
+
new_path = Pathname.new(instance_var_value)
|
91
|
+
instance_variable_set path_instance_var_name, new_path
|
92
|
+
raise RuntimeError, "Must provide a readable file for #{instance_var_name}" unless new_path.readable?
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|