rtbackup 0.1.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile ADDED
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/gem_tasks'
4
+ require 'rspec/core/rake_task'
5
+
6
+ RSpec::Core::RakeTask.new(:spec)
7
+
8
+ task default: :spec
@@ -0,0 +1,83 @@
1
+ ###############################################################################
2
+ #
3
+ # Sample Backup Configuration
4
+ #
5
+ #
6
+ #
7
+ ###############################################################################
8
+
9
+ logging:
10
+ # DEBUG, INFO, WARN, ERROR, FATAL
11
+ log_threshold: 'DEBUG'
12
+ # If set to true, logs to STDOUT and all the subsequent settings in this
13
+ # section will be ignored.
14
+ use_stdout: true
15
+ logfile_path: '/tmp/server_backup.log'
16
+ keep_files: 1
17
+ file_size:
18
+ kilobytes: 100
19
+
20
+ # Optional. Defaults to /var/www
21
+ file_location: 'spec/fixtures/www'
22
+ # The time zone that you'd like backups to be kept in.
23
+ # Backup files will be named with a timestamp located in the given timezone.
24
+ time_zone: 'Singapore'
25
+ # Optional. Will attempt to determine from operating system
26
+ # defaults to the value of the `time_zone` property above.
27
+ system_time_zone: 'Asia/Ho_Chi_Minh'
28
+
29
+ slack:
30
+ # Set up a new webhook at https://slack.com/apps/A0F7XDUAZ-incoming-webhooks
31
+ # Once configured, put the address here.
32
+ webhook: https://hooks.slack.com/services/asdasdasdasdasdasdasdasdasdasd
33
+ # If you want to babysit your backups, set this true.
34
+ notify_on_success: true
35
+ # A list of user ids of persons to @mention when posting success messages.
36
+ mention_users_on_failure:
37
+ - U9ASDFASD
38
+ # - U8SXDFG34
39
+ # - U23SDU3LD
40
+ # - U3234ASD4
41
+ # Leave empty for none.
42
+ mention_users_on_success:
43
+ - U9ASDFASD
44
+
45
+ retention:
46
+ incremental:
47
+ hours: 144
48
+ daily:
49
+ days: 6
50
+ weekly:
51
+ weeks: 3
52
+ monthly:
53
+ months: 11
54
+
55
+ mysql:
56
+ host: 127.0.0.1
57
+ # Database to back up
58
+ database:
59
+ # MySql credentials
60
+ user: root
61
+ password:
62
+ # Path (with trailing slash) to mysql commands e.g. mysqldump
63
+ bin_path: /usr/local/bin/
64
+ # If you are using MySql binary logging:
65
+ # Path to the binary logs, should match the log_bin option in your my.cnf.
66
+ # Comment out if you are not using mysql binary logging
67
+ bin_log: /var/tmp/mysql-bin
68
+
69
+ s3:
70
+ # S3 credentials
71
+ access_key_id: ASDFASDFASDFASDFASDF
72
+ secret_access_key: asdfasdfasdfasdfasdfasdfasdfasdfasdfasdf
73
+ # Bucket in which to store your backups
74
+ # # Bucket in which to store your backups
75
+ bucket: myapp-backup-test
76
+
77
+ # The prefix under which to store and retrieve backups for this server
78
+ # e.g. my_app_name
79
+ prefix: my_test_app
80
+
81
+ # AWS region your bucket lives in.
82
+ # (I suspect you only need to specify this when your 'location' is in a different region.)
83
+ region: ap-southeast-1
data/bin/rtbackup ADDED
@@ -0,0 +1,169 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require 'fileutils'
5
+
6
+ lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
7
+ $LOAD_PATH.unshift lib_dir if File.directory?(lib_dir)
8
+
9
+ require 'server_backups'
10
+ require 'main'
11
+ require 'tmpdir'
12
+
13
+ def find_time_zone(name)
14
+ ActiveSupport::TimeZone.all.select do |zone|
15
+ name.casecmp(zone.name).zero? || name.casecmp(zone.tzinfo.name).zero?
16
+ end.first
17
+ end
18
+
19
+ ERRORS = []
20
+ def backup_thread
21
+ Thread.new do
22
+ begin
23
+ dir = File.join(Dir.getwd, rand.to_s)
24
+ FileUtils.mkdir_p(dir)
25
+ yield dir
26
+ rescue StandardError => error
27
+ ERRORS << error
28
+ ensure
29
+ FileUtils.rm_rf dir
30
+ end
31
+ end
32
+ end
33
+
34
+ def handle_errors(config_path)
35
+ notifier = ServerBackups::Notifier.new(config_path)
36
+ if ERRORS.empty?
37
+ notifier.notify_success
38
+ puts "That's the jam"
39
+ else
40
+ notifier.notify_failure(ERRORS)
41
+ ERRORS.each do |error|
42
+ puts error.message
43
+ puts "\n\n " + error.backtrace.join("\n ")
44
+ end
45
+ end
46
+ end
47
+
48
+ def run_backups(backup_type, config_file, database, db_only, files_only)
49
+ begin
50
+ tmp_dir = File.join(Dir.getwd, rand.to_s)
51
+ FileUtils.mkdir_p tmp_dir
52
+ unless db_only
53
+ ServerBackups::WebsiteBackup.new(config_file,
54
+ tmp_dir, backup_type).do_backup
55
+ end
56
+ unless files_only
57
+ ServerBackups::MysqlBackup.send(backup_type,
58
+ config_file,
59
+ tmp_dir, database).do_backup
60
+ end
61
+ rescue StandardError => error
62
+ puts error.message
63
+ puts "\n\n " + error.backtrace.join("\n ")
64
+ exit(2)
65
+ ensure
66
+ FileUtils.rm_rf tmp_dir
67
+ end
68
+ end
69
+
70
+ def run_restore(config_file, database, up_to, tz, db_only, files_only)
71
+ Chronic.time_class = find_time_zone(tz) if tz
72
+ up_to = Chronic.parse(up_to)
73
+ tmp_dir = File.join(Dir.getwd, rand.to_s)
74
+ begin
75
+ FileUtils.mkdir_p tmp_dir
76
+ unless db_only
77
+ ServerBackups::WebsiteRestore.new(config_file,
78
+ tmp_dir, up_to).do_restore
79
+ end
80
+ unless files_only
81
+ ServerBackups::MysqlRestore.restore(config_file,
82
+ tmp_dir,
83
+ up_to, database)
84
+ end
85
+ rescue StandardError => error
86
+ puts error.message
87
+ puts "\n\n " + error.backtrace.join("\n ")
88
+ exit(2)
89
+ end
90
+ end
91
+
92
+ Main do
93
+ option 'config', 'c' do
94
+ argument :required
95
+ description 'load configuration from YAML file'
96
+ defaults '~/.backup_conf.yml'
97
+ end
98
+
99
+ option 'database', 'd' do
100
+ argument :required
101
+ description 'Which database to back up, defaults to all non-system databases.'
102
+ defaults 'all'
103
+ end
104
+
105
+ option 'db_only', 'b' do
106
+ argument :optional
107
+ description 'Only work with database(s).'
108
+ end
109
+
110
+ option 'files_only', 'f' do
111
+ argument :optional
112
+ description 'Only work with files.'
113
+ end
114
+
115
+ mode 'restore' do
116
+ params[:backup_type].ignore!
117
+ option 'up_to', 'u' do
118
+ argument :required
119
+ description 'The point in time to restore to. See ' \
120
+ 'https://github.com/mojombo/chronic for examples.'
121
+ end
122
+ option 'time_zone', 'z' do
123
+ argument :required
124
+ description 'Time zone that <up_to> is given in. Default: time_zone from config file.'
125
+ end
126
+ def run
127
+ options = params.to_options
128
+ run_restore(*options.slice('config',
129
+ 'database',
130
+ 'up_to',
131
+ 'time_zone',
132
+ 'db_only',
133
+ 'files_only').values)
134
+ end
135
+ end
136
+
137
+ mode 'zones' do
138
+ params[:backup_type].ignore!
139
+ argument 'search' do
140
+ argument :required
141
+ description "Add a regex pattern to search for, e.g. '*america*'"
142
+ defaults '.*'
143
+ end
144
+
145
+ def run
146
+ pattern = Regexp.new(params['search'].value, 'i')
147
+ ActiveSupport::TimeZone.all.sort_by(&:name).each do |timezone|
148
+ if pattern =~ timezone.name || pattern =~ timezone.tzinfo.name
149
+ puts timezone.name + "\t\t" + timezone.tzinfo.name
150
+ end
151
+ end
152
+ end
153
+ end
154
+
155
+ argument 'backup_type' do
156
+ cast :symbol
157
+ validate { |command| ServerBackups::BackupBase::BACKUP_TYPES.include? command }
158
+ description 'specifies the backup type to perform [incremental | daily | weekly | monthly]'
159
+ end
160
+
161
+ def run
162
+ options = params.to_options
163
+ run_backups(*options.slice('backup_type',
164
+ 'config',
165
+ 'database',
166
+ 'db_only',
167
+ 'files_only').values)
168
+ end
169
+ end
data/crontab.txt ADDED
@@ -0,0 +1,4 @@
1
+ 5 0 1 * * server_backup monthly
2
+ 5 0 2-31 * 0 server_backup weekly
3
+ 5 0 2-31 * 1-6 server_backup daily
4
+ 5 1-23 * * * server_backup incremental
Binary file
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'active_support'
4
+ require 'active_support/core_ext'
5
+ require 'chronic'
6
+
7
+ require 'server_backups/version'
8
+ require 'server_backups/errors'
9
+ require 'server_backups/config'
10
+ require 'server_backups/s3'
11
+ require 'server_backups/backup_base'
12
+ require 'server_backups/website_backup'
13
+ require 'server_backups/mysql_backup'
14
+ require 'server_backups/mysql_incremental_backup'
15
+ require 'server_backups/ordered_backup_file_collection'
16
+ require 'server_backups/restore_base'
17
+ require 'server_backups/website_restore'
18
+ require 'server_backups/mysql_restore'
19
+ require 'server_backups/notifier'
20
+
21
+ module ServerBackups
22
+ end
@@ -0,0 +1,110 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'English'
4
+
5
+ module ServerBackups
6
+ class BackupBase
7
+ attr_reader :working_directory, :config, :logger, :s3, :backup_type
8
+
9
+ BACKUP_TYPES = %i[incremental daily weekly monthly].freeze
10
+
11
+ TIMESTAMP_FORMAT = '%Y-%m-%dT%H00.UTC%z'
12
+
13
+ def initialize(config_file, working_directory, backup_type)
14
+ @working_directory = working_directory
15
+ @config = Config.new(config_file)
16
+ Time.zone = config.time_zone
17
+ @logger = config.logger
18
+ @backup_type = backup_type.to_sym
19
+ logger.debug "Initialized #{backup_type} #{self.class.name.demodulize.titleize}, " \
20
+ "prefix: '#{s3_prefix}'"
21
+ end
22
+
23
+ def incremental?
24
+ backup_type == :incremental
25
+ end
26
+
27
+ class << self
28
+ def daily(config_file, working_directory)
29
+ new(config_file, working_directory, :daily)
30
+ end
31
+
32
+ def weekly(config_file, working_directory)
33
+ new(config_file, working_directory, :weekly)
34
+ end
35
+
36
+ def monthly(config_file, working_directory)
37
+ new(config_file, working_directory, :monthly)
38
+ end
39
+
40
+ def incremental(config_file, working_directory)
41
+ new(config_file, working_directory, :incremental)
42
+ end
43
+ end
44
+
45
+ def backup_filename
46
+ "#{self.class.name.demodulize.underscore}.#{backup_type}.#{timestamp}.tgz"
47
+ end
48
+
49
+ def title
50
+ self.class.name.demodulize.titleize
51
+ end
52
+
53
+ def take_backup
54
+ logger.info "Creating #{backup_type} #{title} #{create_archive_command}"
55
+
56
+ system create_archive_command
57
+ unless last_command_succeeded?
58
+ raise BackupCreationError.new("Received #{$CHILD_STATUS} from tar command.",
59
+ self.class, backup_type)
60
+ end
61
+ logger.debug "Backup exited with #{$CHILD_STATUS}"
62
+ end
63
+
64
+ def s3_prefix
65
+ File.join(config.prefix, self.class.name.demodulize.underscore,
66
+ backup_type.to_s, '/')
67
+ end
68
+
69
+ def backup_s3_key
70
+ File.join(s3_prefix, File.basename(backup_filename))
71
+ end
72
+
73
+ def store_backup
74
+ logger.info 'Upload file'
75
+ @uploaded_file = s3.save backup_path, backup_s3_key
76
+ logger.info 'Finished uploading file.'
77
+ end
78
+
79
+ def remove_old_backups
80
+ s3.delete_files_not_newer_than s3_prefix, config.get_retention_threshold(backup_type)
81
+ end
82
+
83
+ def backup_path
84
+ File.join(working_directory, backup_filename)
85
+ end
86
+
87
+ def do_backup
88
+ load_resources
89
+ take_backup
90
+ store_backup
91
+ # verify_backup
92
+ remove_old_backups
93
+ end
94
+
95
+ def timestamp
96
+ Time.zone.now.strftime(TIMESTAMP_FORMAT)
97
+ end
98
+
99
+ def load_resources
100
+ # @mysql = Mys3ql::Mysql.new self.config
101
+ @s3 = S3.new config
102
+ end
103
+
104
+ private
105
+
106
+ def last_command_succeeded?
107
+ $CHILD_STATUS.exitstatus.zero?
108
+ end
109
+ end
110
+ end
@@ -0,0 +1,209 @@
1
+ # frozen_string_literal: true
2
+
3
+ # lifted from https://github.com/airblade/mys3ql/blob/master/lib/mys3ql/config.rb
4
+
5
+ require 'yaml'
6
+ require 'logger'
7
+
8
+ module ServerBackups
9
+ class Config
10
+ attr_reader :logger, :logging, :config_file
11
+ cattr_accessor :config
12
+
13
+ DEFAULT_LOGFILE_SIZE = 1.megabytes
14
+ DEFAULT_LOGFILE_COUNT = 7
15
+
16
+ def initialize(config_file = nil)
17
+ if config_file == '-'
18
+ @config_file = config_file
19
+ @config = Config::config ||= YAML::safe_load(STDIN.read())
20
+ else
21
+ @config_file = config_file || default_config_file
22
+ @config = YAML.load_file File.expand_path(config_file)
23
+ end
24
+
25
+ @logging = @config['logging']
26
+ @logger = Logger.new(log_device, logfile_count, logfile_size)
27
+ rescue Errno::ENOENT => e
28
+ puts e.backtrace
29
+ warn "missing config file #{config_file}"
30
+ exit 1
31
+ end
32
+
33
+ class << self
34
+ def get_time_zone(name)
35
+ ActiveSupport::TimeZone.all.find do |time_zone|
36
+ time_zone.name.casecmp(name).zero? || time_zone.tzinfo.name.casecmp(name).zero?
37
+ end
38
+ end
39
+ end
40
+
41
+ def slack_webhook
42
+ @config.fetch('slack', nil)&.fetch('webhook')
43
+ end
44
+
45
+ def notify_on_success
46
+ @config.fetch('slack', nil)&.fetch('notify_on_success', false)
47
+ end
48
+
49
+ def slack_mention_on_failure
50
+ @config.fetch('slack', nil)&.fetch('mention_users_on_failure', [])
51
+ end
52
+
53
+ def slack_mention_on_success
54
+ @config.fetch('slack', nil)&.fetch('mention_users_on_success', [])
55
+ end
56
+
57
+ #
58
+ # General
59
+ #
60
+ FILE_LOCATION = '/var/www'
61
+
62
+ def web_root
63
+ File.absolute_path(@config['file_location'] || FILE_LOCATION)
64
+ end
65
+
66
+ def time_zone
67
+ @config['time_zone'] || 'UTC'
68
+ end
69
+
70
+ def system_time_zone
71
+ tz = if @config['system_time_zone']
72
+ @config['system_time_zone']
73
+ elsif File.exist?('/etc/timezone')
74
+ File.read('/etc/timezone')
75
+ elsif File.exist?('/etc/localtime')
76
+ %r{([\w_]+\/[\w_]+)$}.match(`ls -l /etc/localtime`).captures.first
77
+ else
78
+ 'UTC'
79
+ end
80
+ self.class.get_time_zone(tz)
81
+ end
82
+
83
+ def log_device
84
+ logging['use_stdout'] == true ? STDOUT : logging['logfile_path']
85
+ end
86
+
87
+ def logfile_size
88
+ if logging['file_size']
89
+ unit, quantity = @logging['file_size'].first
90
+ quantity = quantity.to_i
91
+ quantity.send(unit)
92
+ else
93
+ DEFAULT_LOGFILE_SIZE
94
+ end
95
+ end
96
+
97
+ def logfile_count
98
+ if logging['keep_files']
99
+ logging['keep_files'].to_i
100
+ else
101
+ DEFAULT_LOGFILE_COUNT
102
+ end
103
+ end
104
+
105
+ def get_retention_threshold(backup_type)
106
+ interval, quantity = @config['retention'][backup_type.to_s].first
107
+ quantity = quantity.to_i + 1
108
+ quantity.send(interval).ago
109
+ end
110
+
111
+ def retain_dailies_after
112
+ get_retention_threshold(:daily)
113
+ end
114
+
115
+ def retain_incrementals_after
116
+ get_retention_threshold(:incremental)
117
+ end
118
+
119
+ def retain_monthlies_after
120
+ get_retention_threshold(:monthly)
121
+ end
122
+
123
+ def retain_weeklies_after
124
+ get_retention_threshold(:weekly)
125
+ end
126
+
127
+ #
128
+ # MySQL
129
+ #
130
+
131
+ def db_host
132
+ mysql['host'] || '127.0.0.1'
133
+ end
134
+
135
+ def user
136
+ mysql['user']
137
+ end
138
+
139
+ def password
140
+ mysql['password']
141
+ end
142
+
143
+ def database
144
+ mysql['database']
145
+ end
146
+
147
+ def bin_path
148
+ mysql['bin_path']
149
+ end
150
+
151
+ def bin_log
152
+ mysql['bin_log']
153
+ end
154
+
155
+ MYSQLDUMP = 'mysqldump'
156
+ MYSQL = 'mysql'
157
+ MYSQLBINLOG = 'mysqlbinlog'
158
+
159
+ def mysqldump_bin
160
+ File.join(bin_path, MYSQLDUMP)
161
+ end
162
+
163
+ def mysqlbinlog_bin
164
+ File.join(bin_path, MYSQLBINLOG)
165
+ end
166
+
167
+ def mysql_bin
168
+ File.join(bin_path, MYSQL)
169
+ end
170
+
171
+ #
172
+ # S3
173
+ #
174
+
175
+ def access_key_id
176
+ s3['access_key_id']
177
+ end
178
+
179
+ def secret_access_key
180
+ s3['secret_access_key']
181
+ end
182
+
183
+ def bucket
184
+ s3['bucket']
185
+ end
186
+
187
+ def region
188
+ s3['region']
189
+ end
190
+
191
+ def prefix
192
+ s3['prefix']
193
+ end
194
+
195
+ private
196
+
197
+ def mysql
198
+ @config['mysql']
199
+ end
200
+
201
+ def s3
202
+ @config['s3']
203
+ end
204
+
205
+ def default_config_file
206
+ File.join (ENV['HOME']).to_s, '.mys3ql'
207
+ end
208
+ end
209
+ end