server_backups 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/bin/server_backup ADDED
@@ -0,0 +1,162 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
5
+ $LOAD_PATH.unshift lib_dir if File.directory?(lib_dir)
6
+
7
+ require 'server_backups'
8
+ require 'main'
9
+ require 'tmpdir'
10
+
11
+ def find_time_zone(name)
12
+ ActiveSupport::TimeZone.all.select do |zone|
13
+ name.casecmp(zone.name).zero? || name.casecmp(zone.tzinfo.name).zero?
14
+ end.first
15
+ end
16
+
17
+ ERRORS = []
18
+ def backup_thread
19
+ Thread.new do
20
+ begin
21
+ Dir.mktmpdir do |tmp_dir|
22
+ yield tmp_dir
23
+ end
24
+ rescue StandardError => error
25
+ ERRORS << error
26
+ end
27
+ end
28
+ end
29
+
30
+ def handle_errors(config_path)
31
+ notifier = ServerBackups::Notifier.new(config_path)
32
+ if ERRORS.empty?
33
+ notifier.notify_success
34
+ puts "That's the jam"
35
+ else
36
+ notifier.notify_failure(ERRORS)
37
+ puts "borthnarg" * 15
38
+ puts ERRORS.inspect
39
+ end
40
+ end
41
+
42
+ def run_backups(backup_type, config_file, database, db_only, files_only)
43
+ website_backup = unless db_only
44
+ backup_thread do |tmp_dir|
45
+ ServerBackups::WebsiteBackup.new(config_file,
46
+ tmp_dir, backup_type).do_backup
47
+ end
48
+ end
49
+ database_backup = unless files_only
50
+ backup_thread do |tmp_dir|
51
+ ServerBackups::MysqlBackup.send(backup_type,
52
+ config_file,
53
+ tmp_dir, database).do_backup
54
+ end
55
+ end
56
+ [website_backup, database_backup].compact.each(&:join)
57
+ handle_errors(config_file)
58
+ exit_success!
59
+ end
60
+
61
+ def run_restore(config_file, database, up_to, tz, db_only, files_only)
62
+ Chronic.time_class = find_time_zone(tz) if tz
63
+ up_to = Chronic.parse(up_to)
64
+ website_restore = unless db_only
65
+ backup_thread do |tmp_dir|
66
+ ServerBackups::WebsiteRestore.new(config_file,
67
+ tmp_dir, up_to).do_restore
68
+ end
69
+ end
70
+ database_restore = unless files_only
71
+ backup_thread do |tmp_dir|
72
+ ServerBackups::MysqlRestore.restore(config_file,
73
+ tmp_dir,
74
+ up_to, database)
75
+ end
76
+ end
77
+ [website_restore, database_restore].compact.each(&:join)
78
+ exit_success!
79
+ end
80
+
81
+ def time_zone_match?(pattern, timezone)
82
+ pattern =~ timezone.name || pattern =~ timezone.tzinfo.name
83
+ end
84
+
85
+ Main do
86
+ option 'config', 'c' do
87
+ argument :required
88
+ description 'load configuration from YAML file'
89
+ defaults '~/.backup_conf.yml'
90
+ end
91
+
92
+ option 'database', 'd' do
93
+ argument :required
94
+ description 'Which database to back up, defaults to all non-system databases.'
95
+ defaults 'all'
96
+ end
97
+
98
+ option 'db_only', 'b' do
99
+ argument :optional
100
+ description 'Only work with database(s).'
101
+ end
102
+
103
+ option 'files_only', 'f' do
104
+ argument :optional
105
+ description 'Only work with files.'
106
+ end
107
+
108
+ mode 'restore' do
109
+ params[:backup_type].ignore!
110
+ option 'up_to', 'u' do
111
+ argument :required
112
+ description 'The point in time to restore to. See ' \
113
+ 'https://github.com/mojombo/chronic for examples.'
114
+ end
115
+ option 'time_zone', 'z' do
116
+ argument :required
117
+ description 'Time zone that <up_to> is given in. Default: time_zone from config file.'
118
+ end
119
+ def run
120
+ options = params.to_options
121
+ run_restore(*options.slice('config',
122
+ 'database',
123
+ 'up_to',
124
+ 'time_zone',
125
+ 'db_only',
126
+ 'files_only').values)
127
+ end
128
+ end
129
+
130
+ mode 'zones' do
131
+ params[:backup_type].ignore!
132
+ argument 'search' do
133
+ argument :required
134
+ description "Add a regex pattern to search for, e.g. '*america*'"
135
+ defaults '.*'
136
+ end
137
+
138
+ def run
139
+ pattern = Regexp.new(params['search'].value, 'i')
140
+ ActiveSupport::TimeZone.all.sort_by(&:name).each do |timezone|
141
+ if time_zone_match?(pattern, timezone)
142
+ puts timezone.name + "\t\t" + timezone.tzinfo.name
143
+ end
144
+ end
145
+ end
146
+ end
147
+
148
+ argument 'backup_type' do
149
+ cast :symbol
150
+ validate { |command| ServerBackups::BackupBase::BACKUP_TYPES.include? command }
151
+ description 'specifies the backup type to perform [incremental | daily | weekly | monthly]'
152
+ end
153
+
154
+ def run
155
+ options = params.to_options
156
+ run_backups(*options.slice('backup_type',
157
+ 'config',
158
+ 'database',
159
+ 'db_only',
160
+ 'files_only').values)
161
+ end
162
+ end
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
data/crontab.txt ADDED
@@ -0,0 +1,4 @@
1
+ 5 0 1 * * server_backup monthly
2
+ 5 0 2-31 * 0 server_backup weekly
3
+ 5 0 2-31 * 1-6 server_backup daily
4
+ 5 1-23 * * * server_backup incremental
Binary file
@@ -0,0 +1,110 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'English'
4
+
5
+ module ServerBackups
6
+ class BackupBase
7
+ attr_reader :working_directory, :config, :logger, :s3, :backup_type
8
+
9
+ BACKUP_TYPES = %i[incremental daily weekly monthly].freeze
10
+
11
+ TIMESTAMP_FORMAT = '%Y-%m-%dT%H00.UTC%z'
12
+
13
+ def initialize(config_file, working_directory, backup_type)
14
+ @working_directory = working_directory
15
+ @config = Config.new(config_file)
16
+ Time.zone = config.time_zone
17
+ @logger = config.logger
18
+ @backup_type = backup_type.to_sym
19
+ logger.debug "Initialized #{backup_type} #{self.class.name.demodulize.titleize}, " \
20
+ "prefix: '#{s3_prefix}'"
21
+ end
22
+
23
+ def incremental?
24
+ backup_type == :incremental
25
+ end
26
+
27
+ class << self
28
+ def daily(config_file, working_directory)
29
+ new(config_file, working_directory, :daily)
30
+ end
31
+
32
+ def weekly(config_file, working_directory)
33
+ new(config_file, working_directory, :weekly)
34
+ end
35
+
36
+ def monthly(config_file, working_directory)
37
+ new(config_file, working_directory, :monthly)
38
+ end
39
+
40
+ def incremental(config_file, working_directory)
41
+ new(config_file, working_directory, :incremental)
42
+ end
43
+ end
44
+
45
+ def backup_filename
46
+ "#{self.class.name.demodulize.underscore}.#{backup_type}.#{timestamp}.tgz"
47
+ end
48
+
49
+ def title
50
+ self.class.name.demodulize.titleize
51
+ end
52
+
53
+ def take_backup
54
+ logger.info "Creating #{backup_type} #{title} #{create_archive_command}"
55
+
56
+ system create_archive_command
57
+ unless last_command_succeeded?
58
+ raise BackupCreationError.new("Received #{$CHILD_STATUS} from tar command.",
59
+ self.class, backup_type)
60
+ end
61
+ logger.debug "Backup exited with #{$CHILD_STATUS}"
62
+ end
63
+
64
+ def s3_prefix
65
+ File.join(config.prefix, self.class.name.demodulize.underscore,
66
+ backup_type.to_s, '/')
67
+ end
68
+
69
+ def backup_s3_key
70
+ File.join(s3_prefix, File.basename(backup_filename))
71
+ end
72
+
73
+ def store_backup
74
+ logger.info 'Upload file'
75
+ @uploaded_file = s3.save backup_path, backup_s3_key
76
+ logger.info 'Finished uploading file.'
77
+ end
78
+
79
+ def remove_old_backups
80
+ s3.delete_files_not_newer_than s3_prefix, config.get_retention_threshold(backup_type)
81
+ end
82
+
83
+ def backup_path
84
+ File.join(working_directory, backup_filename)
85
+ end
86
+
87
+ def do_backup
88
+ load_resources
89
+ take_backup
90
+ store_backup
91
+ # verify_backup
92
+ remove_old_backups
93
+ end
94
+
95
+ def timestamp
96
+ Time.zone.now.strftime(TIMESTAMP_FORMAT)
97
+ end
98
+
99
+ def load_resources
100
+ # @mysql = Mys3ql::Mysql.new self.config
101
+ @s3 = S3.new config
102
+ end
103
+
104
+ private
105
+
106
+ def last_command_succeeded?
107
+ $CHILD_STATUS.exitstatus.zero?
108
+ end
109
+ end
110
+ end
@@ -0,0 +1,197 @@
1
+ # frozen_string_literal: true
2
+
3
+ # lifted from https://github.com/airblade/mys3ql/blob/master/lib/mys3ql/config.rb
4
+
5
+ require 'yaml'
6
+ require 'logger'
7
+
8
+ module ServerBackups
9
+ class Config
10
+ attr_reader :logger, :logging, :config_file
11
+
12
+ DEFAULT_LOGFILE_SIZE = 1.megabytes
13
+ DEFAULT_LOGFILE_COUNT = 7
14
+
15
+ def initialize(config_file = nil)
16
+ @config_file = config_file || default_config_file
17
+ @config = YAML.load_file File.expand_path(config_file)
18
+ @logging = @config['logging']
19
+ @logger = Logger.new(log_device, logfile_count, logfile_size)
20
+ rescue Errno::ENOENT
21
+ warn "missing config file #{config_file}"
22
+ exit 1
23
+ end
24
+
25
+ class << self
26
+ def get_time_zone(name)
27
+ ActiveSupport::TimeZone.all.find do |time_zone|
28
+ time_zone.name.casecmp(name).zero? || time_zone.tzinfo.name.casecmp(name).zero?
29
+ end
30
+ end
31
+ end
32
+
33
+ def slack_webhook
34
+ @config.fetch('slack', nil)&.fetch('webhook')
35
+ end
36
+
37
+ def notify_on_success
38
+ @config.fetch('slack', nil)&.fetch('notify_on_success', false)
39
+ end
40
+
41
+ def slack_mention_on_failure
42
+ @config.fetch('slack', nil)&.fetch('mention_users_on_failure', [])
43
+ end
44
+
45
+ def slack_mention_on_success
46
+ @config.fetch('slack', nil)&.fetch('mention_users_on_success', [])
47
+ end
48
+
49
+ #
50
+ # General
51
+ #
52
+ FILE_LOCATION = '/var/www'
53
+
54
+ def web_root
55
+ File.absolute_path(@config['file_location'] || FILE_LOCATION)
56
+ end
57
+
58
+ def time_zone
59
+ @config['time_zone'] || 'UTC'
60
+ end
61
+
62
+ def system_time_zone
63
+ tz = if @config['system_time_zone']
64
+ @config['system_time_zone']
65
+ elsif File.exist?('/etc/timezone')
66
+ File.read('/etc/timezone')
67
+ elsif File.exist?('/etc/localtime')
68
+ %r{([\w_]+\/[\w_]+)$}.match(`ls -l /etc/localtime`).captures.first
69
+ else
70
+ 'UTC'
71
+ end
72
+ self.class.get_time_zone(tz)
73
+ end
74
+
75
+ def log_device
76
+ logging['use_stdout'] == true ? STDOUT : logging['logfile_path']
77
+ end
78
+
79
+ def logfile_size
80
+ if logging['file_size']
81
+ unit, quantity = @logging['file_size'].first
82
+ quantity = quantity.to_i
83
+ quantity.send(unit)
84
+ else
85
+ DEFAULT_LOGFILE_SIZE
86
+ end
87
+ end
88
+
89
+ def logfile_count
90
+ if logging['keep_files']
91
+ logging['keep_files'].to_i
92
+ else
93
+ DEFAULT_LOGFILE_COUNT
94
+ end
95
+ end
96
+
97
+ def get_retention_threshold(backup_type)
98
+ interval, quantity = @config['retention'][backup_type.to_s].first
99
+ quantity = quantity.to_i + 1
100
+ quantity.send(interval).ago
101
+ end
102
+
103
+ def retain_dailies_after
104
+ get_retention_threshold(:daily)
105
+ end
106
+
107
+ def retain_incrementals_after
108
+ get_retention_threshold(:incremental)
109
+ end
110
+
111
+ def retain_monthlies_after
112
+ get_retention_threshold(:monthly)
113
+ end
114
+
115
+ def retain_weeklies_after
116
+ get_retention_threshold(:weekly)
117
+ end
118
+
119
+ #
120
+ # MySQL
121
+ #
122
+
123
+ def user
124
+ mysql['user']
125
+ end
126
+
127
+ def password
128
+ mysql['password']
129
+ end
130
+
131
+ def database
132
+ mysql['database']
133
+ end
134
+
135
+ def bin_path
136
+ mysql['bin_path']
137
+ end
138
+
139
+ def bin_log
140
+ mysql['bin_log']
141
+ end
142
+
143
+ MYSQLDUMP = 'mysqldump'
144
+ MYSQL = 'mysql'
145
+ MYSQLBINLOG = 'mysqlbinlog'
146
+
147
+ def mysqldump_bin
148
+ File.join(bin_path, MYSQLDUMP)
149
+ end
150
+
151
+ def mysqlbinlog_bin
152
+ File.join(bin_path, MYSQLBINLOG)
153
+ end
154
+
155
+ def mysql_bin
156
+ File.join(bin_path, MYSQL)
157
+ end
158
+
159
+ #
160
+ # S3
161
+ #
162
+
163
+ def access_key_id
164
+ s3['access_key_id']
165
+ end
166
+
167
+ def secret_access_key
168
+ s3['secret_access_key']
169
+ end
170
+
171
+ def bucket
172
+ s3['bucket']
173
+ end
174
+
175
+ def region
176
+ s3['region']
177
+ end
178
+
179
+ def prefix
180
+ s3['prefix']
181
+ end
182
+
183
+ private
184
+
185
+ def mysql
186
+ @config['mysql']
187
+ end
188
+
189
+ def s3
190
+ @config['s3']
191
+ end
192
+
193
+ def default_config_file
194
+ File.join (ENV['HOME']).to_s, '.mys3ql'
195
+ end
196
+ end
197
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServerBackups
4
+ class BackupCreationError < StandardError
5
+ attr_reader :backup_class, :backup_type
6
+ def initialize(msg, backup_class, backup_type)
7
+ @backup_class = backup_class
8
+ @backup_type = backup_type
9
+ super(msg)
10
+ end
11
+ end
12
+
13
+ class RestoreTarError < BackupCreationError; end
14
+ end
@@ -0,0 +1,90 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServerBackups
4
+ class MysqlBackup < BackupBase
5
+ attr_reader :database_name
6
+
7
+ SYSTEM_DATABASES = %w[sys information_schema mysql performance_schema].freeze
8
+
9
+ def initialize(config_file, working_directory, backup_type, database_name)
10
+ @database_name = database_name
11
+ super(config_file, working_directory, backup_type)
12
+ end
13
+
14
+ def do_backup
15
+ if database_name == 'all'
16
+ backup_all_databases
17
+ else
18
+ super
19
+ end
20
+ end
21
+
22
+ def backup_all_databases
23
+ @database_name = 'mysql'
24
+ all_databases.each do |database|
25
+ self.class.send(backup_type,
26
+ config.config_file,
27
+ working_directory,
28
+ database).do_backup
29
+ end
30
+ end
31
+
32
+ class << self
33
+ def daily(config_file, working_directory, database_name)
34
+ new(config_file, working_directory, :daily, database_name)
35
+ end
36
+
37
+ def weekly(config_file, working_directory, database_name)
38
+ new(config_file, working_directory, :weekly, database_name)
39
+ end
40
+
41
+ def monthly(config_file, working_directory, database_name)
42
+ new(config_file, working_directory, :monthly, database_name)
43
+ end
44
+
45
+ def incremental(config_file, working_directory, database_name)
46
+ MysqlIncrementalBackup.new(config_file, working_directory, database_name)
47
+ end
48
+ end
49
+
50
+ def create_archive_command
51
+ cmd = config.mysqldump_bin + ' --quick --single-transaction --create-options '
52
+ cmd += ' --flush-logs --master-data=2 --delete-master-logs ' if binary_logging?
53
+ cmd + cli_options + ' | gzip > ' + backup_path
54
+ end
55
+
56
+ def s3_prefix
57
+ File.join(config.prefix, self.class.name.demodulize.underscore,
58
+ database_name, backup_type.to_s, '/')
59
+ end
60
+
61
+ def backup_filename
62
+ "mysql_backup.#{backup_type}.#{timestamp}.sql.gz"
63
+ end
64
+
65
+ def all_databases
66
+ execute_sql('show databases;').reject do |db_name|
67
+ db_name.in?(SYSTEM_DATABASES)
68
+ end
69
+ end
70
+
71
+ private
72
+
73
+ def binary_logging?
74
+ !config.bin_log.blank?
75
+ end
76
+
77
+ def cli_options
78
+ cmd = config.password.blank? ? '' : " -p'#{config.password}' "
79
+ cmd + " -u'#{config.user}' " + database_name
80
+ end
81
+
82
+ def execute_sql(sql)
83
+ cmd = "#{config.mysql_bin} --silent --skip-column-names -e \"#{sql}\" #{cli_options}"
84
+ logger.debug "Executing raw SQL against #{database_name}\n#{cmd}"
85
+ output = `#{cmd}`
86
+ logger.debug "Returned #{$CHILD_STATUS.inspect}. STDOUT was:\n#{output}"
87
+ output.split("\n") unless output.blank?
88
+ end
89
+ end
90
+ end
@@ -0,0 +1,89 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServerBackups
4
+ class MysqlIncrementalBackup < MysqlBackup
5
+ def initialize(config_file, working_directory, database_name)
6
+ database_name = 'mysql' if database_name == 'all'
7
+
8
+ super(config_file, working_directory, :incremental, database_name)
9
+ end
10
+
11
+ class BinlogFilename
12
+ attr_reader :path
13
+ def initialize(path)
14
+ @path = path
15
+ end
16
+
17
+ def log_index
18
+ /(\d{6})/.match(File.basename(path)).captures.first
19
+ end
20
+ end
21
+
22
+ def do_backup
23
+ load_resources
24
+ flush_logs
25
+ each_bin_log do |file|
26
+ index = BinlogFilename.new(file).log_index
27
+ next if index.in?(already_stored_log_indexes)
28
+ backup_single_bin_log(file)
29
+ end
30
+ end
31
+
32
+ def s3_prefix
33
+ File.join(config.prefix, 'mysql_backup', 'incremental', '/')
34
+ end
35
+
36
+ def flush_logs
37
+ execute_sql('flush logs;')
38
+ end
39
+
40
+ def each_bin_log
41
+ # execute 'flush logs'
42
+ logs = Dir.glob("#{config.bin_log}.[0-9]*").sort_by { |f| f[/\d+/].to_i }
43
+ logs_to_backup = logs[0..-2] # all logs except the last, which is in use
44
+ logs_to_backup.each do |log_file|
45
+ yield log_file
46
+ end
47
+ end
48
+
49
+ # def each_remote_bin_log
50
+ # remote_bin_logs.each do |file|
51
+ # yield(**parse_remote_binlog_filename(file))
52
+ # end
53
+ # end
54
+
55
+ # def parse_remote_binlog_filename(file)
56
+ # filename = File.basename(file.key)
57
+ # prefix, index, timestamp = REMOTE_FILENAME_REGEX.match(filename).captures
58
+ # {
59
+ # key: file.key,
60
+ # file: file,
61
+ # prefix: prefix,
62
+ # index: index,
63
+ # timestamp: timestamp,
64
+ # datetime: Time.zone.strptime(timestamp, TIMESTAMP_FORMAT)
65
+ # }
66
+ # end
67
+
68
+ private
69
+
70
+ REMOTE_FILENAME_REGEX = /(.*)\.(\d+)\.(.{15})/
71
+ def already_stored_log_indexes
72
+ remote_bin_logs.map do |s3object|
73
+ _, index = REMOTE_FILENAME_REGEX.match(s3object.key).captures
74
+ index
75
+ end
76
+ end
77
+
78
+ def remote_bin_logs
79
+ s3.bucket.objects(prefix: s3_prefix)
80
+ end
81
+
82
+ def backup_single_bin_log(file)
83
+ logger.debug "Backing up #{file}."
84
+ dest_filename = File.basename(file) + '.' + timestamp
85
+ logger.info "Storing #{file} to #{dest_filename}"
86
+ s3.save file, File.join(s3_prefix, dest_filename)
87
+ end
88
+ end
89
+ end