server_backups 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'tempfile'
4
+
5
+ module ServerBackups
6
+ class MysqlRestore < RestoreBase
7
+ ALL_DATABASES = 'all'
8
+
9
+ def initialize(config_file, working_dir, restore_point, database)
10
+ @database = database
11
+ super config_file, working_dir, restore_point
12
+ end
13
+
14
+ def full_backup_file
15
+ full_backup_prefix = File.join(config.prefix, 'mysql_backup', database)
16
+ s3.get_ordered_collection(full_backup_prefix).full_backup_for(restore_point)
17
+ end
18
+
19
+ def incremental_backups
20
+ incr_backup_prefix = File.join(config.prefix, 'mysql_backup')
21
+ s3.get_ordered_collection(incr_backup_prefix).incremental_backups_for(restore_point)
22
+ end
23
+
24
+ def restore_script_path
25
+ File.join(working_dir, "#{database}.sql")
26
+ end
27
+
28
+ ETC_TIMEZONE = '/etc/timezone'
29
+
30
+ def formatted_restore_point_in_system_time_zone
31
+ restore_point.in_time_zone(config.system_time_zone) \
32
+ .strftime('%Y-%m-%d %H:%M:%S')
33
+ end
34
+
35
+ def do_restore
36
+ full_backup_file.get response_target: (restore_script_path + '.gz')
37
+ system "gunzip #{restore_script_path}.gz"
38
+
39
+ incremental_backups.each do |s3object|
40
+ file = Tempfile.new('foo')
41
+ begin
42
+ s3object.get response_target: file
43
+ file.close
44
+ system config.mysqlbinlog_bin + ' ' + file.path + \
45
+ " --stop-datetime='#{formatted_restore_point_in_system_time_zone}'" \
46
+ " --database=#{database} >> " + restore_script_path
47
+ ensure
48
+ file.close
49
+ file.unlink # deletes the temp file
50
+ end
51
+ end
52
+
53
+ execute_script restore_script_path
54
+ end
55
+ # #{@config.bin_path}mysqlbinlog --database=#{@config.database} #{file}
56
+ class << self
57
+ def restore(config_file, working_dir, restore_point, database)
58
+ return new(config_file, working_dir, restore_point, database).do_restore \
59
+ if database != ALL_DATABASES
60
+
61
+ all_databases(config_file, working_dir).each do |db_name|
62
+ new(config_file, working_dir, restore_point, db_name).do_restore
63
+ end
64
+ end
65
+
66
+ def all_databases(config_file, working_dir)
67
+ MysqlBackup.new(config_file, working_dir, 'daily', 'mysql').all_databases
68
+ end
69
+ end
70
+
71
+ private
72
+
73
+ def cli_options
74
+ cmd = config.password.blank? ? '' : " -p'#{config.password}' "
75
+ cmd + " -u'#{config.user}' " + database
76
+ end
77
+
78
+ def execute_script(path)
79
+ cmd = "#{config.mysql_bin} --silent --skip-column-names #{cli_options}"
80
+ logger.debug "Executing raw SQL against #{ database}\n#{cmd}"
81
+ output = `#{cmd} < #{path}`
82
+ logger.debug "Returned #{$CHILD_STATUS.inspect}. STDOUT was:\n#{output}"
83
+ output.split("\n") unless output.blank?
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,33 @@
1
+ require 'slack-notifier'
2
+
3
+ module ServerBackups
4
+ class Notifier
5
+ attr_reader :config
6
+
7
+ def initialize(config_path)
8
+ @config = Config.new(config_path)
9
+ end
10
+
11
+ def notify_success
12
+ return unless config.slack_webhook && config.notify_on_success
13
+
14
+ notifier = Slack::Notifier.new config.slack_webhook
15
+ message = "Backups at `#{config.prefix}` succeeded. "
16
+ message += config.slack_mention_on_success.map{|t| "<@#{t}>"}.to_sentence
17
+ notifier.post text: message, icon_emoji: ':100:'
18
+ end
19
+
20
+ def notify_failure(errors)
21
+ return unless config.slack_webhook
22
+
23
+ notifier = Slack::Notifier.new config.slack_webhook
24
+ message = "Backups at `#{config.prefix}` failed. "
25
+ message += config.slack_mention_on_failure.map{|t| "<@#{t}>"}.to_sentence
26
+ attachments = []
27
+ for error in errors do
28
+ attachments << {text: error.message + "\n" + error.backtrace.join("\n")}
29
+ end
30
+ notifier.post text: message, icon_emoji: ':bomb:', attachments: attachments
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServerBackups
4
+ class OrderedBackupFileCollection
5
+ attr_reader :s3_collection
6
+ def initialize(s3_collection)
7
+ @s3_collection = s3_collection
8
+ end
9
+
10
+ def full_backup_for(restore_point)
11
+ sorted(full_backups).reverse.find do |file|
12
+ get_timestamp_from_s3_object(file) <= restore_point
13
+ end
14
+ end
15
+
16
+ def incremental_backups_for(restore_point)
17
+ sorted eligible_incremental_backups(restore_point)
18
+ end
19
+
20
+ INCREMENTAL = /incremental/i
21
+ def full_backups
22
+ s3_collection.reject { |file| INCREMENTAL =~ file.key }
23
+ end
24
+
25
+ def incremental_backups
26
+ @incremental_backups ||=
27
+ sorted(s3_collection.select { |file| INCREMENTAL =~ file.key }).to_a
28
+ end
29
+
30
+ private
31
+
32
+ TIMESTAMP_REGEXP = /(\d{4})-(\d{2})-(\d{2})T(\d{2})00\.UTC([+-]\d{4})/
33
+ def get_timestamp_from_s3_object(file)
34
+ time_parts = TIMESTAMP_REGEXP.match(file.key).captures
35
+ time_parts[-1].insert(3, ':')
36
+ # Add in hours and seconds arguments
37
+ # https://ruby-doc.org/core-2.2.0/Time.html#method-c-new
38
+ time_parts.insert(4, 0, 0)
39
+ Time.new(*time_parts)
40
+ end
41
+
42
+ def sorted(coll)
43
+ coll.sort_by { |file| get_timestamp_from_s3_object file }
44
+ end
45
+
46
+ def eligible_incremental_backups(restore_point)
47
+ full_backup_timestamp = get_timestamp_from_s3_object full_backup_for(restore_point)
48
+ incremental_backups.select do |file|
49
+ get_timestamp_from_s3_object(file) > full_backup_timestamp &&
50
+ get_timestamp_from_s3_object(file) <= restore_point
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServerBackups
4
+ class RestoreBase
5
+ attr_reader :config, :s3, :working_dir, :restore_point, :database
6
+
7
+ def initialize(config_file, working_dir, restore_point)
8
+ @working_dir = working_dir
9
+ @config = Config.new(config_file)
10
+ Time.zone = config.time_zone
11
+ @restore_point = if restore_point.present?
12
+ restore_point
13
+ else
14
+ Time.zone.now
15
+ end
16
+ @s3 = S3.new(config)
17
+ logger.debug "Initialized #{title}."
18
+ end
19
+
20
+ private
21
+
22
+ def title
23
+ self.class.name.demodulize.titleize
24
+ end
25
+
26
+ TIMESTAMP_REGEXP = /(\d{4})-(\d{2})-(\d{2})T(\d{2})00\.UTC([+-]\d{4})/
27
+ def extract_backup_time_from_filename(filename)
28
+ time_parts = TIMESTAMP_REGEXP.match(filename).captures
29
+ # Add in hours and seconds arguments
30
+ # https://ruby-doc.org/core-2.2.0/Time.html#method-c-new
31
+ time_parts.insert(4, 0, 0)
32
+ Time.new(*time_parts)
33
+ end
34
+
35
+ def all_files
36
+ @all_files ||= s3.get_ordered_collection(s3_prefix)
37
+ end
38
+
39
+ def logger
40
+ config.logger
41
+ end
42
+
43
+ def last_command_succeeded?
44
+ $CHILD_STATUS.exitstatus.zero?
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,71 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-s3'
4
+
5
+ module ServerBackups
6
+ class S3
7
+ PROVIDER = 'AWS'
8
+ attr_reader :config, :logger
9
+ def initialize(config)
10
+ @config = config
11
+ @logger = config.logger
12
+ end
13
+
14
+ def client
15
+ @client ||= begin
16
+ Aws.config[:credentials] = Aws::Credentials.new(
17
+ config.access_key_id, config.secret_access_key
18
+ )
19
+ Aws::S3::Client.new region: config.region
20
+ end
21
+ end
22
+
23
+ def bucket
24
+ @bucket ||= Aws::S3::Bucket.new(config.bucket, client: client)
25
+ end
26
+
27
+ def get_ordered_collection(prefix)
28
+ OrderedBackupFileCollection.new bucket.objects(prefix: prefix)
29
+ end
30
+
31
+ def delete_files_not_newer_than(key, age)
32
+ bucket.objects(prefix: key).each do |file|
33
+ destroy key, true unless file.last_modified.to_datetime > age
34
+ end
35
+ end
36
+
37
+ def exists?(path)
38
+ logger.debug "Exists? #{config.bucket} #{path}"
39
+ !bucket.objects(prefix: path).to_a.empty?
40
+ # !!client.head_object(bucket: config.bucket, key: path)
41
+ end
42
+
43
+ def destroy(key, existence_known = false)
44
+ return unless existence_known || exists?(key)
45
+ client.delete_object bucket: config.bucket, key: key
46
+ end
47
+
48
+ def save(local_file_name, s3_key)
49
+ full_path = if s3_key[-1] == '/'
50
+ File.join(s3_key, File.basename(local_file_name))
51
+ else
52
+ s3_key
53
+ end
54
+
55
+ return if exists?(full_path)
56
+ file = Aws::S3::Object.new(config.bucket, full_path, client: client)
57
+ file.put(
58
+ acl: 'private',
59
+ body: File.open(local_file_name, 'rb'),
60
+ content_md5: md5of(local_file_name),
61
+ storage_class: 'STANDARD_IA'
62
+ )
63
+ end
64
+
65
+ private
66
+
67
+ def md5of(local_file_name)
68
+ Digest::MD5.base64digest(File.read(local_file_name))
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServerBackups
4
+ VERSION = '0.1.8'
5
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ServerBackups
4
+ class WebsiteBackup < BackupBase
5
+ SNAPSHOT_PATH = Pathname('~/.var.www.backup.tar.snapshot.bin').expand_path
6
+
7
+ def restore(time); end
8
+
9
+ def backup_filename
10
+ "#{self.class.name.demodulize.underscore}.#{backup_type}.#{timestamp}.tgz"
11
+ end
12
+
13
+ def create_archive_command
14
+ <<-CMD
15
+ tar --create -C '#{config.web_root}' --listed-incremental=#{SNAPSHOT_PATH} --gzip \
16
+ --no-check-device \
17
+ #{'--level=0' unless incremental?} \
18
+ --file=#{backup_path} #{config.web_root}
19
+ CMD
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'English'
4
+
5
+ module ServerBackups
6
+ class WebsiteRestore < RestoreBase
7
+ def files_to_restore
8
+ [all_files.full_backup_for(restore_point),
9
+ *all_files.incremental_backups_for(restore_point)]
10
+ end
11
+
12
+ def do_restore
13
+ logger.warn "Moving old #{config.web_root} ---->>> #{config.web_root}.backup\n" \
14
+ 'You will have to delete it yourself.'
15
+ make_copy_of_existing_web_root
16
+ files_to_restore.each_with_index { |file, index| restore_file(file, index) }
17
+ end
18
+
19
+ private
20
+
21
+ def make_copy_of_existing_web_root
22
+ FileUtils.mv config.web_root, "#{config.web_root}.backup" \
23
+ if File.exist?(config.web_root)
24
+ FileUtils.mkdir config.web_root
25
+ end
26
+
27
+ def s3_prefix
28
+ File.join(config.prefix, 'website_backup', '/')
29
+ end
30
+
31
+ def restore_file(s3object, ordinal)
32
+ local_filename = File.join(working_dir, "#{ordinal}.tgz")
33
+ s3object.get response_target: local_filename
34
+ system untar_command(local_filename)
35
+ unless last_command_succeeded?
36
+ raise BackupCreationError.new("Received #{$CHILD_STATUS} from tar command.",
37
+ self.class, 'restore')
38
+ end
39
+ logger.info "Restored #{File.basename(s3object.key)}"
40
+ end
41
+
42
+ def untar_command(local_filename)
43
+ <<-CMD
44
+ tar zxp -C '/' --listed-incremental=/dev/null \
45
+ --no-check-device \
46
+ --file=#{local_filename}
47
+ CMD
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'active_support'
4
+ require 'active_support/core_ext'
5
+ require 'chronic'
6
+
7
+ require 'server_backups/version'
8
+ require 'server_backups/errors'
9
+ require 'server_backups/config'
10
+ require 'server_backups/s3'
11
+ require 'server_backups/backup_base'
12
+ require 'server_backups/website_backup'
13
+ require 'server_backups/mysql_backup'
14
+ require 'server_backups/mysql_incremental_backup'
15
+ require 'server_backups/ordered_backup_file_collection'
16
+ require 'server_backups/restore_base'
17
+ require 'server_backups/website_restore'
18
+ require 'server_backups/mysql_restore'
19
+ require 'server_backups/notifier'
20
+
21
+ module ServerBackups
22
+ end
@@ -0,0 +1,45 @@
1
+
2
+ # frozen_string_literal: true
3
+
4
+ lib = File.expand_path('lib', __dir__)
5
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
6
+ require 'server_backups/version'
7
+
8
+ Gem::Specification.new do |spec|
9
+ spec.name = 'server_backups'
10
+ spec.version = ServerBackups::VERSION
11
+ spec.authors = ['Tyler Gannon']
12
+ spec.email = ['tgannon@gmail.com']
13
+
14
+ spec.summary = 'For taking backups of servers.'
15
+ spec.description = 'For taking backups of servers.'
16
+ spec.homepage = 'https://github.com/tylergannon/server_backups'
17
+ spec.license = 'MIT'
18
+
19
+ # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
20
+ # to allow pushing to a single host or delete this section to allow pushing to any host.
21
+ if spec.respond_to?(:metadata)
22
+ spec.metadata['allowed_push_host'] = "https://rubygems.org"
23
+ else
24
+ raise 'RubyGems 2.0 or newer is required to protect against ' \
25
+ 'public gem pushes.'
26
+ end
27
+
28
+ spec.add_dependency 'activesupport', '~> 5.1'
29
+ spec.add_dependency 'aws-sdk-s3', '~> 1.8'
30
+ spec.add_dependency 'main', '~> 6.2'
31
+ spec.add_dependency 'slack-notifier', '~> 2.3'
32
+
33
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
34
+ f.match(%r{^(test|spec|features)/})
35
+ end
36
+ spec.bindir = 'bin'
37
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
38
+ spec.require_paths = ['lib']
39
+
40
+ spec.add_development_dependency 'bundler', '~> 1.16'
41
+ spec.add_development_dependency 'rake', '~> 10.5'
42
+ spec.add_development_dependency 'rspec', '~> 3.0'
43
+ spec.add_development_dependency 'vcr', '~> 4.0'
44
+ spec.add_development_dependency 'webmock', '~> 3.3'
45
+ end
metadata ADDED
@@ -0,0 +1,207 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: server_backups
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.8
5
+ platform: ruby
6
+ authors:
7
+ - Tyler Gannon
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2018-05-26 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: activesupport
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '5.1'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '5.1'
27
+ - !ruby/object:Gem::Dependency
28
+ name: aws-sdk-s3
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.8'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.8'
41
+ - !ruby/object:Gem::Dependency
42
+ name: main
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '6.2'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '6.2'
55
+ - !ruby/object:Gem::Dependency
56
+ name: slack-notifier
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '2.3'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '2.3'
69
+ - !ruby/object:Gem::Dependency
70
+ name: bundler
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '1.16'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '1.16'
83
+ - !ruby/object:Gem::Dependency
84
+ name: rake
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '10.5'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '10.5'
97
+ - !ruby/object:Gem::Dependency
98
+ name: rspec
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '3.0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '3.0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: vcr
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '4.0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '4.0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: webmock
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '3.3'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '3.3'
139
+ description: For taking backups of servers.
140
+ email:
141
+ - tgannon@gmail.com
142
+ executables:
143
+ - console
144
+ - server_backup
145
+ - setup
146
+ extensions: []
147
+ extra_rdoc_files: []
148
+ files:
149
+ - ".gitignore"
150
+ - ".rspec"
151
+ - ".rubocop.yml"
152
+ - ".rubocop_todo.yml"
153
+ - ".ruby-version"
154
+ - ".travis.yml"
155
+ - CODE_OF_CONDUCT.md
156
+ - Gemfile
157
+ - Gemfile.lock
158
+ - LICENSE.txt
159
+ - README.md
160
+ - Rakefile
161
+ - backup_conf.sample.yml
162
+ - bin/console
163
+ - bin/server_backup
164
+ - bin/setup
165
+ - crontab.txt
166
+ - incremental_snapshot
167
+ - lib/server_backups.rb
168
+ - lib/server_backups/backup_base.rb
169
+ - lib/server_backups/config.rb
170
+ - lib/server_backups/errors.rb
171
+ - lib/server_backups/mysql_backup.rb
172
+ - lib/server_backups/mysql_incremental_backup.rb
173
+ - lib/server_backups/mysql_restore.rb
174
+ - lib/server_backups/notifier.rb
175
+ - lib/server_backups/ordered_backup_file_collection.rb
176
+ - lib/server_backups/restore_base.rb
177
+ - lib/server_backups/s3.rb
178
+ - lib/server_backups/version.rb
179
+ - lib/server_backups/website_backup.rb
180
+ - lib/server_backups/website_restore.rb
181
+ - server_backups.gemspec
182
+ homepage: https://github.com/tylergannon/server_backups
183
+ licenses:
184
+ - MIT
185
+ metadata:
186
+ allowed_push_host: https://rubygems.org
187
+ post_install_message:
188
+ rdoc_options: []
189
+ require_paths:
190
+ - lib
191
+ required_ruby_version: !ruby/object:Gem::Requirement
192
+ requirements:
193
+ - - ">="
194
+ - !ruby/object:Gem::Version
195
+ version: '0'
196
+ required_rubygems_version: !ruby/object:Gem::Requirement
197
+ requirements:
198
+ - - ">="
199
+ - !ruby/object:Gem::Version
200
+ version: '0'
201
+ requirements: []
202
+ rubyforge_project:
203
+ rubygems_version: 2.5.1
204
+ signing_key:
205
+ specification_version: 4
206
+ summary: For taking backups of servers.
207
+ test_files: []