scalingo_backups_manager 0.4.0 → 0.6.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/lib/scalingo_backups_manager/cli.rb +33 -11
- data/lib/scalingo_backups_manager/notification.rb +30 -0
- data/lib/scalingo_backups_manager/restore/mysql.rb +1 -1
- data/lib/scalingo_backups_manager/restore/postgres.rb +14 -8
- data/lib/scalingo_backups_manager/sftp_tools.rb +28 -2
- data/lib/scalingo_backups_manager/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7be1ca03d45192dc394492ef8fb5c0a8b852ca08b4f96bbc4469960fda01e2f6
|
4
|
+
data.tar.gz: 721b03b36837bff777bbc9e79fdc85e188fa03524fac0405c92844209ba2fa59
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4015ae6b481e82b6a565caa4db86738df7a819875241e405cc2ca742874ef8d09d3f68e50d13459a7cb341c07b4813aed998978a950afbafb6b97f6a4c115732
|
7
|
+
data.tar.gz: 405ef4f1c59acd60099b9a5bd71246e906d181d86399ba998b4290f5dce976210ffe02b420aebb4f0a514975de13674aa007e54050efa60650e6e7a52a14c724
|
data/Gemfile.lock
CHANGED
@@ -128,40 +128,62 @@ module ScalingoBackupsManager
|
|
128
128
|
def upload_to_ftp
|
129
129
|
invoke :download, [], application: options[:application], addon: options[:addon]
|
130
130
|
configuration = Configuration.new
|
131
|
+
opts = {
|
132
|
+
webhooks: configuration.config[:webhooks]
|
133
|
+
}
|
131
134
|
configuration.for_each_addons do |application, addon|
|
132
|
-
|
135
|
+
step = 1
|
136
|
+
sftp_config = addon.sftp_config
|
133
137
|
path = ("#{addon.config[:path]}" || "backups/#{addon.addon_provider[:id]}") + "/#{Time.now.strftime("%Y%m%d")}.tar.gz"
|
134
138
|
next unless File.exists?(path)
|
139
|
+
puts "** Upload backup for #{application.name} **"
|
135
140
|
|
136
|
-
sftp = ScalingoBackupsManager::SftpTools.new(
|
141
|
+
sftp = ScalingoBackupsManager::SftpTools.new(sftp_config[:auth])
|
137
142
|
|
138
143
|
folders = [
|
139
|
-
|
140
|
-
|
144
|
+
sftp_config.dig(:auth, :dir),
|
145
|
+
sftp_config.dig(:dir) || application.name,
|
141
146
|
addon.addon_provider[:id]
|
142
147
|
]
|
143
148
|
|
144
|
-
if
|
149
|
+
if sftp_config[:retention].blank?
|
145
150
|
remote_path = "/" + [folders].delete_if(&:blank?).join("/")
|
146
151
|
sftp.mkdir!(remote_path)
|
147
|
-
sftp.upload_file(path, remote_path)
|
152
|
+
sftp.upload_file(path, remote_path, options: opts)
|
148
153
|
next
|
149
154
|
end
|
150
155
|
|
151
|
-
|
152
|
-
|
153
|
-
|
156
|
+
sftp_config[:retention].each do |k, retention_config|
|
157
|
+
retention_folders = folders.dup
|
158
|
+
retention_folders << sftp_config.dig(:retention, k, :dir)
|
159
|
+
remote_path = "/" + retention_folders.delete_if(&:blank?).join("/")
|
160
|
+
puts "#{step} - Creating remote directory at #{remote_path}"
|
161
|
+
step += 1
|
154
162
|
sftp.mkdir!(remote_path)
|
155
163
|
case k
|
156
164
|
when "daily"
|
157
|
-
sftp.upload_file(path, remote_path)
|
165
|
+
sftp.upload_file(path, remote_path, options: opts)
|
166
|
+
files = sftp.list_files(remote_path)
|
167
|
+
puts "#{step} - Checking daily backups"
|
168
|
+
step += 1
|
169
|
+
if files.size > retention_config[:ttl]
|
170
|
+
files_to_remove = files.sort_by(&:name).shift(files.size - retention_config[:ttl])
|
171
|
+
puts "#{step} - Removing #{files_to_remove.size} backups because of ttl configuration"
|
172
|
+
files_to_remove.each do |file|
|
173
|
+
puts "Removing file #{remote_path + "/" + file.name}"
|
174
|
+
sftp.remove!(remote_path + "/" + file.name)
|
175
|
+
end
|
176
|
+
end
|
158
177
|
when "monthly"
|
159
178
|
next unless Date.today.day == 1
|
160
|
-
sftp.upload_file(path, remote_path)
|
179
|
+
sftp.upload_file(path, remote_path, options: opts)
|
180
|
+
puts "#{step} - Checking monthly backups"
|
181
|
+
step += 1
|
161
182
|
end
|
162
183
|
end
|
163
184
|
|
164
185
|
end
|
186
|
+
FileUtils.rm_r 'backups/'
|
165
187
|
end
|
166
188
|
|
167
189
|
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'httparty'
|
2
|
+
module ScalingoBackupsManager
|
3
|
+
|
4
|
+
class Notification
|
5
|
+
|
6
|
+
def self.send_slack_notification(hook_url, message)
|
7
|
+
HTTParty.post(
|
8
|
+
hook_url,
|
9
|
+
body: {
|
10
|
+
message: message
|
11
|
+
}.to_json,
|
12
|
+
headers: { 'Content-Type' => 'application/json' }
|
13
|
+
)
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.send_discord_notification(hook_url, message)
|
17
|
+
payload = {
|
18
|
+
user: 'Scalingo backups manager',
|
19
|
+
content: message
|
20
|
+
}.to_json
|
21
|
+
HTTParty.post(
|
22
|
+
hook_url,
|
23
|
+
body: payload,
|
24
|
+
headers: { 'Content-Type': 'application/json' }
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
@@ -63,7 +63,7 @@ module ScalingoBackupsManager
|
|
63
63
|
puts "*** Restoring backup to mysql database ***"
|
64
64
|
puts "Command: #{restore_cmd}"
|
65
65
|
system(restore_cmd)
|
66
|
-
|
66
|
+
FileUtils.rm_r destination_path unless opts[:skip_rm]
|
67
67
|
end
|
68
68
|
|
69
69
|
end
|
@@ -34,21 +34,27 @@ module ScalingoBackupsManager
|
|
34
34
|
user: rails_db_config["user"],
|
35
35
|
}
|
36
36
|
|
37
|
-
restore_cmd = "/usr/bin/env
|
37
|
+
restore_cmd = "/usr/bin/env pg_restore"
|
38
38
|
|
39
|
+
file_path = Dir["#{destination_path}*.pgsql"]
|
40
|
+
if file_path.empty?
|
41
|
+
puts "*** No SQL file found in tar ***"
|
42
|
+
return
|
43
|
+
end
|
44
|
+
restore_cmd << " #{file_path.first}"
|
45
|
+
|
46
|
+
if config[:host].present?
|
47
|
+
restore_cmd << " -h #{opts[:host] || config[:host] || 'localhost'}"
|
48
|
+
end
|
39
49
|
if config[:user].present?
|
40
|
-
restore_cmd << "
|
41
|
-
if config[:password].present?
|
42
|
-
restore_cmd << " --password"
|
43
|
-
restore_cmd << " #{config[:password]}"
|
44
|
-
end
|
50
|
+
restore_cmd << " -U #{config[:user]}"
|
45
51
|
end
|
46
52
|
|
47
|
-
if config[:port].present?
|
53
|
+
if opts[:port].present? || config[:port].present?
|
48
54
|
restore_cmd << " -p #{opts[:port] || config[:port] || 5432}"
|
49
55
|
end
|
50
56
|
|
51
|
-
restore_cmd << "
|
57
|
+
restore_cmd << " -d #{config[:database]}"
|
52
58
|
|
53
59
|
puts "*** Restoring backup to Postgres database ***"
|
54
60
|
system(restore_cmd)
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'scalingo_backups_manager/notification'
|
1
2
|
require 'net/sftp'
|
2
3
|
|
3
4
|
module ScalingoBackupsManager
|
@@ -14,6 +15,22 @@ module ScalingoBackupsManager
|
|
14
15
|
end
|
15
16
|
end
|
16
17
|
|
18
|
+
def list_files(path)
|
19
|
+
files = []
|
20
|
+
start do |sftp|
|
21
|
+
sftp.dir.glob("#{path}", "*.tar.gz").each do |file|
|
22
|
+
files << file
|
23
|
+
end
|
24
|
+
end
|
25
|
+
files
|
26
|
+
end
|
27
|
+
|
28
|
+
def remove!(path)
|
29
|
+
start do |sftp|
|
30
|
+
sftp.remove!(path)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
17
34
|
def mkdir!(path)
|
18
35
|
start do |sftp|
|
19
36
|
folder_tree = []
|
@@ -29,10 +46,19 @@ module ScalingoBackupsManager
|
|
29
46
|
end
|
30
47
|
end
|
31
48
|
|
32
|
-
def upload_file(filepath, remote_dir)
|
49
|
+
def upload_file(filepath, remote_dir, options: {})
|
33
50
|
filename = filepath.split('/').last
|
34
51
|
start do |sftp|
|
35
|
-
|
52
|
+
begin
|
53
|
+
sftp.upload!(filepath, "#{remote_dir}/#{filename}")
|
54
|
+
rescue
|
55
|
+
if options.dig(:webhooks, :slack_webhook_url)
|
56
|
+
ScalingoBackupsManager::Notification.send_slack_notification(options.dig(:webhooks, :slack_webhook_url), "An error has occured while uploading backup, see the logs for more information")
|
57
|
+
end
|
58
|
+
if options.dig(:webhooks, :discord_webhook_url)
|
59
|
+
ScalingoBackupsManager::Notification.send_discord_notification(options.dig(:webhooks, :discord_webhook_url), "An error has occured while uploading backup, see the logs for more information")
|
60
|
+
end
|
61
|
+
end
|
36
62
|
end
|
37
63
|
end
|
38
64
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: scalingo_backups_manager
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kevin Clercin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-06-
|
11
|
+
date: 2021-06-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -92,6 +92,7 @@ files:
|
|
92
92
|
- lib/scalingo_backups_manager/backup.rb
|
93
93
|
- lib/scalingo_backups_manager/cli.rb
|
94
94
|
- lib/scalingo_backups_manager/configuration.rb
|
95
|
+
- lib/scalingo_backups_manager/notification.rb
|
95
96
|
- lib/scalingo_backups_manager/restore/mongodb.rb
|
96
97
|
- lib/scalingo_backups_manager/restore/mysql.rb
|
97
98
|
- lib/scalingo_backups_manager/restore/postgres.rb
|