server_backups 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +13 -0
- data/.rspec +3 -0
- data/.rubocop.yml +15 -0
- data/.rubocop_todo.yml +35 -0
- data/.ruby-version +1 -0
- data/.travis.yml +5 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +8 -0
- data/Gemfile.lock +89 -0
- data/LICENSE.txt +21 -0
- data/README.md +167 -0
- data/Rakefile +8 -0
- data/backup_conf.sample.yml +82 -0
- data/bin/console +15 -0
- data/bin/server_backup +162 -0
- data/bin/setup +8 -0
- data/crontab.txt +4 -0
- data/incremental_snapshot +0 -0
- data/lib/server_backups/backup_base.rb +110 -0
- data/lib/server_backups/config.rb +197 -0
- data/lib/server_backups/errors.rb +14 -0
- data/lib/server_backups/mysql_backup.rb +90 -0
- data/lib/server_backups/mysql_incremental_backup.rb +89 -0
- data/lib/server_backups/mysql_restore.rb +86 -0
- data/lib/server_backups/notifier.rb +33 -0
- data/lib/server_backups/ordered_backup_file_collection.rb +54 -0
- data/lib/server_backups/restore_base.rb +47 -0
- data/lib/server_backups/s3.rb +71 -0
- data/lib/server_backups/version.rb +5 -0
- data/lib/server_backups/website_backup.rb +22 -0
- data/lib/server_backups/website_restore.rb +50 -0
- data/lib/server_backups.rb +22 -0
- data/server_backups.gemspec +45 -0
- metadata +207 -0
data/bin/server_backup
ADDED
@@ -0,0 +1,162 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
|
5
|
+
$LOAD_PATH.unshift lib_dir if File.directory?(lib_dir)
|
6
|
+
|
7
|
+
require 'server_backups'
|
8
|
+
require 'main'
|
9
|
+
require 'tmpdir'
|
10
|
+
|
11
|
+
def find_time_zone(name)
|
12
|
+
ActiveSupport::TimeZone.all.select do |zone|
|
13
|
+
name.casecmp(zone.name).zero? || name.casecmp(zone.tzinfo.name).zero?
|
14
|
+
end.first
|
15
|
+
end
|
16
|
+
|
17
|
+
ERRORS = []
|
18
|
+
def backup_thread
|
19
|
+
Thread.new do
|
20
|
+
begin
|
21
|
+
Dir.mktmpdir do |tmp_dir|
|
22
|
+
yield tmp_dir
|
23
|
+
end
|
24
|
+
rescue StandardError => error
|
25
|
+
ERRORS << error
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def handle_errors(config_path)
|
31
|
+
notifier = ServerBackups::Notifier.new(config_path)
|
32
|
+
if ERRORS.empty?
|
33
|
+
notifier.notify_success
|
34
|
+
puts "That's the jam"
|
35
|
+
else
|
36
|
+
notifier.notify_failure(ERRORS)
|
37
|
+
puts "borthnarg" * 15
|
38
|
+
puts ERRORS.inspect
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def run_backups(backup_type, config_file, database, db_only, files_only)
|
43
|
+
website_backup = unless db_only
|
44
|
+
backup_thread do |tmp_dir|
|
45
|
+
ServerBackups::WebsiteBackup.new(config_file,
|
46
|
+
tmp_dir, backup_type).do_backup
|
47
|
+
end
|
48
|
+
end
|
49
|
+
database_backup = unless files_only
|
50
|
+
backup_thread do |tmp_dir|
|
51
|
+
ServerBackups::MysqlBackup.send(backup_type,
|
52
|
+
config_file,
|
53
|
+
tmp_dir, database).do_backup
|
54
|
+
end
|
55
|
+
end
|
56
|
+
[website_backup, database_backup].compact.each(&:join)
|
57
|
+
handle_errors(config_file)
|
58
|
+
exit_success!
|
59
|
+
end
|
60
|
+
|
61
|
+
def run_restore(config_file, database, up_to, tz, db_only, files_only)
|
62
|
+
Chronic.time_class = find_time_zone(tz) if tz
|
63
|
+
up_to = Chronic.parse(up_to)
|
64
|
+
website_restore = unless db_only
|
65
|
+
backup_thread do |tmp_dir|
|
66
|
+
ServerBackups::WebsiteRestore.new(config_file,
|
67
|
+
tmp_dir, up_to).do_restore
|
68
|
+
end
|
69
|
+
end
|
70
|
+
database_restore = unless files_only
|
71
|
+
backup_thread do |tmp_dir|
|
72
|
+
ServerBackups::MysqlRestore.restore(config_file,
|
73
|
+
tmp_dir,
|
74
|
+
up_to, database)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
[website_restore, database_restore].compact.each(&:join)
|
78
|
+
exit_success!
|
79
|
+
end
|
80
|
+
|
81
|
+
def time_zone_match?(pattern, timezone)
|
82
|
+
pattern =~ timezone.name || pattern =~ timezone.tzinfo.name
|
83
|
+
end
|
84
|
+
|
85
|
+
Main do
|
86
|
+
option 'config', 'c' do
|
87
|
+
argument :required
|
88
|
+
description 'load configuration from YAML file'
|
89
|
+
defaults '~/.backup_conf.yml'
|
90
|
+
end
|
91
|
+
|
92
|
+
option 'database', 'd' do
|
93
|
+
argument :required
|
94
|
+
description 'Which database to back up, defaults to all non-system databases.'
|
95
|
+
defaults 'all'
|
96
|
+
end
|
97
|
+
|
98
|
+
option 'db_only', 'b' do
|
99
|
+
argument :optional
|
100
|
+
description 'Only work with database(s).'
|
101
|
+
end
|
102
|
+
|
103
|
+
option 'files_only', 'f' do
|
104
|
+
argument :optional
|
105
|
+
description 'Only work with files.'
|
106
|
+
end
|
107
|
+
|
108
|
+
mode 'restore' do
|
109
|
+
params[:backup_type].ignore!
|
110
|
+
option 'up_to', 'u' do
|
111
|
+
argument :required
|
112
|
+
description 'The point in time to restore to. See ' \
|
113
|
+
'https://github.com/mojombo/chronic for examples.'
|
114
|
+
end
|
115
|
+
option 'time_zone', 'z' do
|
116
|
+
argument :required
|
117
|
+
description 'Time zone that <up_to> is given in. Default: time_zone from config file.'
|
118
|
+
end
|
119
|
+
def run
|
120
|
+
options = params.to_options
|
121
|
+
run_restore(*options.slice('config',
|
122
|
+
'database',
|
123
|
+
'up_to',
|
124
|
+
'time_zone',
|
125
|
+
'db_only',
|
126
|
+
'files_only').values)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
mode 'zones' do
|
131
|
+
params[:backup_type].ignore!
|
132
|
+
argument 'search' do
|
133
|
+
argument :required
|
134
|
+
description "Add a regex pattern to search for, e.g. '*america*'"
|
135
|
+
defaults '.*'
|
136
|
+
end
|
137
|
+
|
138
|
+
def run
|
139
|
+
pattern = Regexp.new(params['search'].value, 'i')
|
140
|
+
ActiveSupport::TimeZone.all.sort_by(&:name).each do |timezone|
|
141
|
+
if time_zone_match?(pattern, timezone)
|
142
|
+
puts timezone.name + "\t\t" + timezone.tzinfo.name
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
argument 'backup_type' do
|
149
|
+
cast :symbol
|
150
|
+
validate { |command| ServerBackups::BackupBase::BACKUP_TYPES.include? command }
|
151
|
+
description 'specifies the backup type to perform [incremental | daily | weekly | monthly]'
|
152
|
+
end
|
153
|
+
|
154
|
+
def run
|
155
|
+
options = params.to_options
|
156
|
+
run_backups(*options.slice('backup_type',
|
157
|
+
'config',
|
158
|
+
'database',
|
159
|
+
'db_only',
|
160
|
+
'files_only').values)
|
161
|
+
end
|
162
|
+
end
|
data/bin/setup
ADDED
data/crontab.txt
ADDED
Binary file
|
@@ -0,0 +1,110 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'English'
|
4
|
+
|
5
|
+
module ServerBackups
|
6
|
+
class BackupBase
|
7
|
+
attr_reader :working_directory, :config, :logger, :s3, :backup_type
|
8
|
+
|
9
|
+
BACKUP_TYPES = %i[incremental daily weekly monthly].freeze
|
10
|
+
|
11
|
+
TIMESTAMP_FORMAT = '%Y-%m-%dT%H00.UTC%z'
|
12
|
+
|
13
|
+
def initialize(config_file, working_directory, backup_type)
|
14
|
+
@working_directory = working_directory
|
15
|
+
@config = Config.new(config_file)
|
16
|
+
Time.zone = config.time_zone
|
17
|
+
@logger = config.logger
|
18
|
+
@backup_type = backup_type.to_sym
|
19
|
+
logger.debug "Initialized #{backup_type} #{self.class.name.demodulize.titleize}, " \
|
20
|
+
"prefix: '#{s3_prefix}'"
|
21
|
+
end
|
22
|
+
|
23
|
+
def incremental?
|
24
|
+
backup_type == :incremental
|
25
|
+
end
|
26
|
+
|
27
|
+
class << self
|
28
|
+
def daily(config_file, working_directory)
|
29
|
+
new(config_file, working_directory, :daily)
|
30
|
+
end
|
31
|
+
|
32
|
+
def weekly(config_file, working_directory)
|
33
|
+
new(config_file, working_directory, :weekly)
|
34
|
+
end
|
35
|
+
|
36
|
+
def monthly(config_file, working_directory)
|
37
|
+
new(config_file, working_directory, :monthly)
|
38
|
+
end
|
39
|
+
|
40
|
+
def incremental(config_file, working_directory)
|
41
|
+
new(config_file, working_directory, :incremental)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def backup_filename
|
46
|
+
"#{self.class.name.demodulize.underscore}.#{backup_type}.#{timestamp}.tgz"
|
47
|
+
end
|
48
|
+
|
49
|
+
def title
|
50
|
+
self.class.name.demodulize.titleize
|
51
|
+
end
|
52
|
+
|
53
|
+
def take_backup
|
54
|
+
logger.info "Creating #{backup_type} #{title} #{create_archive_command}"
|
55
|
+
|
56
|
+
system create_archive_command
|
57
|
+
unless last_command_succeeded?
|
58
|
+
raise BackupCreationError.new("Received #{$CHILD_STATUS} from tar command.",
|
59
|
+
self.class, backup_type)
|
60
|
+
end
|
61
|
+
logger.debug "Backup exited with #{$CHILD_STATUS}"
|
62
|
+
end
|
63
|
+
|
64
|
+
def s3_prefix
|
65
|
+
File.join(config.prefix, self.class.name.demodulize.underscore,
|
66
|
+
backup_type.to_s, '/')
|
67
|
+
end
|
68
|
+
|
69
|
+
def backup_s3_key
|
70
|
+
File.join(s3_prefix, File.basename(backup_filename))
|
71
|
+
end
|
72
|
+
|
73
|
+
def store_backup
|
74
|
+
logger.info 'Upload file'
|
75
|
+
@uploaded_file = s3.save backup_path, backup_s3_key
|
76
|
+
logger.info 'Finished uploading file.'
|
77
|
+
end
|
78
|
+
|
79
|
+
def remove_old_backups
|
80
|
+
s3.delete_files_not_newer_than s3_prefix, config.get_retention_threshold(backup_type)
|
81
|
+
end
|
82
|
+
|
83
|
+
def backup_path
|
84
|
+
File.join(working_directory, backup_filename)
|
85
|
+
end
|
86
|
+
|
87
|
+
def do_backup
|
88
|
+
load_resources
|
89
|
+
take_backup
|
90
|
+
store_backup
|
91
|
+
# verify_backup
|
92
|
+
remove_old_backups
|
93
|
+
end
|
94
|
+
|
95
|
+
def timestamp
|
96
|
+
Time.zone.now.strftime(TIMESTAMP_FORMAT)
|
97
|
+
end
|
98
|
+
|
99
|
+
def load_resources
|
100
|
+
# @mysql = Mys3ql::Mysql.new self.config
|
101
|
+
@s3 = S3.new config
|
102
|
+
end
|
103
|
+
|
104
|
+
private
|
105
|
+
|
106
|
+
def last_command_succeeded?
|
107
|
+
$CHILD_STATUS.exitstatus.zero?
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
@@ -0,0 +1,197 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# lifted from https://github.com/airblade/mys3ql/blob/master/lib/mys3ql/config.rb
|
4
|
+
|
5
|
+
require 'yaml'
|
6
|
+
require 'logger'
|
7
|
+
|
8
|
+
module ServerBackups
|
9
|
+
class Config
|
10
|
+
attr_reader :logger, :logging, :config_file
|
11
|
+
|
12
|
+
DEFAULT_LOGFILE_SIZE = 1.megabytes
|
13
|
+
DEFAULT_LOGFILE_COUNT = 7
|
14
|
+
|
15
|
+
def initialize(config_file = nil)
|
16
|
+
@config_file = config_file || default_config_file
|
17
|
+
@config = YAML.load_file File.expand_path(config_file)
|
18
|
+
@logging = @config['logging']
|
19
|
+
@logger = Logger.new(log_device, logfile_count, logfile_size)
|
20
|
+
rescue Errno::ENOENT
|
21
|
+
warn "missing config file #{config_file}"
|
22
|
+
exit 1
|
23
|
+
end
|
24
|
+
|
25
|
+
class << self
|
26
|
+
def get_time_zone(name)
|
27
|
+
ActiveSupport::TimeZone.all.find do |time_zone|
|
28
|
+
time_zone.name.casecmp(name).zero? || time_zone.tzinfo.name.casecmp(name).zero?
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def slack_webhook
|
34
|
+
@config.fetch('slack', nil)&.fetch('webhook')
|
35
|
+
end
|
36
|
+
|
37
|
+
def notify_on_success
|
38
|
+
@config.fetch('slack', nil)&.fetch('notify_on_success', false)
|
39
|
+
end
|
40
|
+
|
41
|
+
def slack_mention_on_failure
|
42
|
+
@config.fetch('slack', nil)&.fetch('mention_users_on_failure', [])
|
43
|
+
end
|
44
|
+
|
45
|
+
def slack_mention_on_success
|
46
|
+
@config.fetch('slack', nil)&.fetch('mention_users_on_success', [])
|
47
|
+
end
|
48
|
+
|
49
|
+
#
|
50
|
+
# General
|
51
|
+
#
|
52
|
+
FILE_LOCATION = '/var/www'
|
53
|
+
|
54
|
+
def web_root
|
55
|
+
File.absolute_path(@config['file_location'] || FILE_LOCATION)
|
56
|
+
end
|
57
|
+
|
58
|
+
def time_zone
|
59
|
+
@config['time_zone'] || 'UTC'
|
60
|
+
end
|
61
|
+
|
62
|
+
def system_time_zone
|
63
|
+
tz = if @config['system_time_zone']
|
64
|
+
@config['system_time_zone']
|
65
|
+
elsif File.exist?('/etc/timezone')
|
66
|
+
File.read('/etc/timezone')
|
67
|
+
elsif File.exist?('/etc/localtime')
|
68
|
+
%r{([\w_]+\/[\w_]+)$}.match(`ls -l /etc/localtime`).captures.first
|
69
|
+
else
|
70
|
+
'UTC'
|
71
|
+
end
|
72
|
+
self.class.get_time_zone(tz)
|
73
|
+
end
|
74
|
+
|
75
|
+
def log_device
|
76
|
+
logging['use_stdout'] == true ? STDOUT : logging['logfile_path']
|
77
|
+
end
|
78
|
+
|
79
|
+
def logfile_size
|
80
|
+
if logging['file_size']
|
81
|
+
unit, quantity = @logging['file_size'].first
|
82
|
+
quantity = quantity.to_i
|
83
|
+
quantity.send(unit)
|
84
|
+
else
|
85
|
+
DEFAULT_LOGFILE_SIZE
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def logfile_count
|
90
|
+
if logging['keep_files']
|
91
|
+
logging['keep_files'].to_i
|
92
|
+
else
|
93
|
+
DEFAULT_LOGFILE_COUNT
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def get_retention_threshold(backup_type)
|
98
|
+
interval, quantity = @config['retention'][backup_type.to_s].first
|
99
|
+
quantity = quantity.to_i + 1
|
100
|
+
quantity.send(interval).ago
|
101
|
+
end
|
102
|
+
|
103
|
+
def retain_dailies_after
|
104
|
+
get_retention_threshold(:daily)
|
105
|
+
end
|
106
|
+
|
107
|
+
def retain_incrementals_after
|
108
|
+
get_retention_threshold(:incremental)
|
109
|
+
end
|
110
|
+
|
111
|
+
def retain_monthlies_after
|
112
|
+
get_retention_threshold(:monthly)
|
113
|
+
end
|
114
|
+
|
115
|
+
def retain_weeklies_after
|
116
|
+
get_retention_threshold(:weekly)
|
117
|
+
end
|
118
|
+
|
119
|
+
#
|
120
|
+
# MySQL
|
121
|
+
#
|
122
|
+
|
123
|
+
def user
|
124
|
+
mysql['user']
|
125
|
+
end
|
126
|
+
|
127
|
+
def password
|
128
|
+
mysql['password']
|
129
|
+
end
|
130
|
+
|
131
|
+
def database
|
132
|
+
mysql['database']
|
133
|
+
end
|
134
|
+
|
135
|
+
def bin_path
|
136
|
+
mysql['bin_path']
|
137
|
+
end
|
138
|
+
|
139
|
+
def bin_log
|
140
|
+
mysql['bin_log']
|
141
|
+
end
|
142
|
+
|
143
|
+
MYSQLDUMP = 'mysqldump'
|
144
|
+
MYSQL = 'mysql'
|
145
|
+
MYSQLBINLOG = 'mysqlbinlog'
|
146
|
+
|
147
|
+
def mysqldump_bin
|
148
|
+
File.join(bin_path, MYSQLDUMP)
|
149
|
+
end
|
150
|
+
|
151
|
+
def mysqlbinlog_bin
|
152
|
+
File.join(bin_path, MYSQLBINLOG)
|
153
|
+
end
|
154
|
+
|
155
|
+
def mysql_bin
|
156
|
+
File.join(bin_path, MYSQL)
|
157
|
+
end
|
158
|
+
|
159
|
+
#
|
160
|
+
# S3
|
161
|
+
#
|
162
|
+
|
163
|
+
def access_key_id
|
164
|
+
s3['access_key_id']
|
165
|
+
end
|
166
|
+
|
167
|
+
def secret_access_key
|
168
|
+
s3['secret_access_key']
|
169
|
+
end
|
170
|
+
|
171
|
+
def bucket
|
172
|
+
s3['bucket']
|
173
|
+
end
|
174
|
+
|
175
|
+
def region
|
176
|
+
s3['region']
|
177
|
+
end
|
178
|
+
|
179
|
+
def prefix
|
180
|
+
s3['prefix']
|
181
|
+
end
|
182
|
+
|
183
|
+
private
|
184
|
+
|
185
|
+
def mysql
|
186
|
+
@config['mysql']
|
187
|
+
end
|
188
|
+
|
189
|
+
def s3
|
190
|
+
@config['s3']
|
191
|
+
end
|
192
|
+
|
193
|
+
def default_config_file
|
194
|
+
File.join (ENV['HOME']).to_s, '.mys3ql'
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ServerBackups
|
4
|
+
class BackupCreationError < StandardError
|
5
|
+
attr_reader :backup_class, :backup_type
|
6
|
+
def initialize(msg, backup_class, backup_type)
|
7
|
+
@backup_class = backup_class
|
8
|
+
@backup_type = backup_type
|
9
|
+
super(msg)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
class RestoreTarError < BackupCreationError; end
|
14
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ServerBackups
|
4
|
+
class MysqlBackup < BackupBase
|
5
|
+
attr_reader :database_name
|
6
|
+
|
7
|
+
SYSTEM_DATABASES = %w[sys information_schema mysql performance_schema].freeze
|
8
|
+
|
9
|
+
def initialize(config_file, working_directory, backup_type, database_name)
|
10
|
+
@database_name = database_name
|
11
|
+
super(config_file, working_directory, backup_type)
|
12
|
+
end
|
13
|
+
|
14
|
+
def do_backup
|
15
|
+
if database_name == 'all'
|
16
|
+
backup_all_databases
|
17
|
+
else
|
18
|
+
super
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def backup_all_databases
|
23
|
+
@database_name = 'mysql'
|
24
|
+
all_databases.each do |database|
|
25
|
+
self.class.send(backup_type,
|
26
|
+
config.config_file,
|
27
|
+
working_directory,
|
28
|
+
database).do_backup
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
class << self
|
33
|
+
def daily(config_file, working_directory, database_name)
|
34
|
+
new(config_file, working_directory, :daily, database_name)
|
35
|
+
end
|
36
|
+
|
37
|
+
def weekly(config_file, working_directory, database_name)
|
38
|
+
new(config_file, working_directory, :weekly, database_name)
|
39
|
+
end
|
40
|
+
|
41
|
+
def monthly(config_file, working_directory, database_name)
|
42
|
+
new(config_file, working_directory, :monthly, database_name)
|
43
|
+
end
|
44
|
+
|
45
|
+
def incremental(config_file, working_directory, database_name)
|
46
|
+
MysqlIncrementalBackup.new(config_file, working_directory, database_name)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def create_archive_command
|
51
|
+
cmd = config.mysqldump_bin + ' --quick --single-transaction --create-options '
|
52
|
+
cmd += ' --flush-logs --master-data=2 --delete-master-logs ' if binary_logging?
|
53
|
+
cmd + cli_options + ' | gzip > ' + backup_path
|
54
|
+
end
|
55
|
+
|
56
|
+
def s3_prefix
|
57
|
+
File.join(config.prefix, self.class.name.demodulize.underscore,
|
58
|
+
database_name, backup_type.to_s, '/')
|
59
|
+
end
|
60
|
+
|
61
|
+
def backup_filename
|
62
|
+
"mysql_backup.#{backup_type}.#{timestamp}.sql.gz"
|
63
|
+
end
|
64
|
+
|
65
|
+
def all_databases
|
66
|
+
execute_sql('show databases;').reject do |db_name|
|
67
|
+
db_name.in?(SYSTEM_DATABASES)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
private
|
72
|
+
|
73
|
+
def binary_logging?
|
74
|
+
!config.bin_log.blank?
|
75
|
+
end
|
76
|
+
|
77
|
+
def cli_options
|
78
|
+
cmd = config.password.blank? ? '' : " -p'#{config.password}' "
|
79
|
+
cmd + " -u'#{config.user}' " + database_name
|
80
|
+
end
|
81
|
+
|
82
|
+
def execute_sql(sql)
|
83
|
+
cmd = "#{config.mysql_bin} --silent --skip-column-names -e \"#{sql}\" #{cli_options}"
|
84
|
+
logger.debug "Executing raw SQL against #{database_name}\n#{cmd}"
|
85
|
+
output = `#{cmd}`
|
86
|
+
logger.debug "Returned #{$CHILD_STATUS.inspect}. STDOUT was:\n#{output}"
|
87
|
+
output.split("\n") unless output.blank?
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module ServerBackups
|
4
|
+
class MysqlIncrementalBackup < MysqlBackup
|
5
|
+
def initialize(config_file, working_directory, database_name)
|
6
|
+
database_name = 'mysql' if database_name == 'all'
|
7
|
+
|
8
|
+
super(config_file, working_directory, :incremental, database_name)
|
9
|
+
end
|
10
|
+
|
11
|
+
class BinlogFilename
|
12
|
+
attr_reader :path
|
13
|
+
def initialize(path)
|
14
|
+
@path = path
|
15
|
+
end
|
16
|
+
|
17
|
+
def log_index
|
18
|
+
/(\d{6})/.match(File.basename(path)).captures.first
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def do_backup
|
23
|
+
load_resources
|
24
|
+
flush_logs
|
25
|
+
each_bin_log do |file|
|
26
|
+
index = BinlogFilename.new(file).log_index
|
27
|
+
next if index.in?(already_stored_log_indexes)
|
28
|
+
backup_single_bin_log(file)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def s3_prefix
|
33
|
+
File.join(config.prefix, 'mysql_backup', 'incremental', '/')
|
34
|
+
end
|
35
|
+
|
36
|
+
def flush_logs
|
37
|
+
execute_sql('flush logs;')
|
38
|
+
end
|
39
|
+
|
40
|
+
def each_bin_log
|
41
|
+
# execute 'flush logs'
|
42
|
+
logs = Dir.glob("#{config.bin_log}.[0-9]*").sort_by { |f| f[/\d+/].to_i }
|
43
|
+
logs_to_backup = logs[0..-2] # all logs except the last, which is in use
|
44
|
+
logs_to_backup.each do |log_file|
|
45
|
+
yield log_file
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
# def each_remote_bin_log
|
50
|
+
# remote_bin_logs.each do |file|
|
51
|
+
# yield(**parse_remote_binlog_filename(file))
|
52
|
+
# end
|
53
|
+
# end
|
54
|
+
|
55
|
+
# def parse_remote_binlog_filename(file)
|
56
|
+
# filename = File.basename(file.key)
|
57
|
+
# prefix, index, timestamp = REMOTE_FILENAME_REGEX.match(filename).captures
|
58
|
+
# {
|
59
|
+
# key: file.key,
|
60
|
+
# file: file,
|
61
|
+
# prefix: prefix,
|
62
|
+
# index: index,
|
63
|
+
# timestamp: timestamp,
|
64
|
+
# datetime: Time.zone.strptime(timestamp, TIMESTAMP_FORMAT)
|
65
|
+
# }
|
66
|
+
# end
|
67
|
+
|
68
|
+
private
|
69
|
+
|
70
|
+
REMOTE_FILENAME_REGEX = /(.*)\.(\d+)\.(.{15})/
|
71
|
+
def already_stored_log_indexes
|
72
|
+
remote_bin_logs.map do |s3object|
|
73
|
+
_, index = REMOTE_FILENAME_REGEX.match(s3object.key).captures
|
74
|
+
index
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def remote_bin_logs
|
79
|
+
s3.bucket.objects(prefix: s3_prefix)
|
80
|
+
end
|
81
|
+
|
82
|
+
def backup_single_bin_log(file)
|
83
|
+
logger.debug "Backing up #{file}."
|
84
|
+
dest_filename = File.basename(file) + '.' + timestamp
|
85
|
+
logger.info "Storing #{file} to #{dest_filename}"
|
86
|
+
s3.save file, File.join(s3_prefix, dest_filename)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|