scalingo_backups_manager 0.3.0 → 0.6.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +1 -0
- data/Gemfile.lock +5 -1
- data/README.md +0 -1
- data/lib/scalingo_backups_manager/addon.rb +5 -4
- data/lib/scalingo_backups_manager/cli.rb +64 -0
- data/lib/scalingo_backups_manager/configuration.rb +9 -5
- data/lib/scalingo_backups_manager/notification.rb +30 -0
- data/lib/scalingo_backups_manager/restore/postgres.rb +1 -1
- data/lib/scalingo_backups_manager/sftp_tools.rb +66 -0
- data/lib/scalingo_backups_manager/version.rb +1 -1
- data/scalingo_backups_manager.gemspec +1 -0
- metadata +18 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bec014dcdbd0fe4aa8a6e646b131b43f7af832aa3a13d22f24900d9dcfeff857
|
4
|
+
data.tar.gz: 1758ce4b459fe4448ddeaeda52302e94b48083685148d398061bc4be96211574
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7b7a1f1f853aa11ea8135effcc15df970aedf269145cccb471a430705f1130e71919116f93ed1f9b2c2062a141191f214d9c99bfe5bc5f3a806abf591453b3a0
|
7
|
+
data.tar.gz: d313ddc959d7757050d9276e79a516395e119dc32de69278736d350a31a0ac1581a1c41b1b490ba1b842801710ba93e4985dd469312a7a0e5c88448b2494fe98
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,8 +1,9 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
scalingo_backups_manager (0.
|
4
|
+
scalingo_backups_manager (0.6.0)
|
5
5
|
httparty (~> 0.18)
|
6
|
+
net-sftp (~> 3.0.0)
|
6
7
|
scalingo (~> 3.0)
|
7
8
|
thor (~> 1.1)
|
8
9
|
|
@@ -33,6 +34,9 @@ GEM
|
|
33
34
|
multi_json (1.15.0)
|
34
35
|
multi_xml (0.6.0)
|
35
36
|
multipart-post (2.1.1)
|
37
|
+
net-sftp (3.0.0)
|
38
|
+
net-ssh (>= 5.0.0, < 7.0.0)
|
39
|
+
net-ssh (6.1.0)
|
36
40
|
rake (12.3.3)
|
37
41
|
rspec (3.10.0)
|
38
42
|
rspec-core (~> 3.10.0)
|
data/README.md
CHANGED
@@ -5,19 +5,20 @@ module ScalingoBackupsManager
|
|
5
5
|
|
6
6
|
class Addon
|
7
7
|
|
8
|
-
attr_accessor :application, :addon, :config
|
8
|
+
attr_accessor :application, :addon, :config, :sftp_config
|
9
9
|
|
10
10
|
DEFAULT_DATABASE_API_ROOT_URL = "https://db-api.osc-fr1.scalingo.com"
|
11
11
|
|
12
|
-
def self.find(app, id, config: {})
|
12
|
+
def self.find(app, id, config: {}, sftp_config: {})
|
13
13
|
addon = Configuration.client.addons.find(app.id, id).data
|
14
|
-
self.new(app, addon, config: config)
|
14
|
+
self.new(app, addon, config: config, sftp_config: sftp_config)
|
15
15
|
end
|
16
16
|
|
17
|
-
def initialize(app, addon, config: {})
|
17
|
+
def initialize(app, addon, config: {}, sftp_config: {})
|
18
18
|
raise "Application must be set" unless app && app.is_a?(ScalingoBackupsManager::Application)
|
19
19
|
@application = app
|
20
20
|
@addon = addon
|
21
|
+
@sftp_config = sftp_config
|
21
22
|
@config = config
|
22
23
|
end
|
23
24
|
|
@@ -5,6 +5,7 @@ require 'scalingo_backups_manager/addon'
|
|
5
5
|
require 'scalingo_backups_manager/restore/mongodb'
|
6
6
|
require 'scalingo_backups_manager/restore/postgres'
|
7
7
|
require 'scalingo_backups_manager/restore/mysql'
|
8
|
+
require 'scalingo_backups_manager/sftp_tools'
|
8
9
|
|
9
10
|
module ScalingoBackupsManager
|
10
11
|
|
@@ -83,6 +84,7 @@ module ScalingoBackupsManager
|
|
83
84
|
backup = backups.first
|
84
85
|
download_link = backup.download_link
|
85
86
|
if download_link
|
87
|
+
puts "Downloading #{application.name} last backup"
|
86
88
|
path = ("#{addon.config[:path]}" || "backups/#{addon.addon_provider[:id]}") + "/#{Time.now.strftime("%Y%m%d")}.tar.gz"
|
87
89
|
if File.exist?(path)
|
88
90
|
puts "Backup already download, skipping..."
|
@@ -122,6 +124,68 @@ module ScalingoBackupsManager
|
|
122
124
|
end
|
123
125
|
end
|
124
126
|
|
127
|
+
desc "upload_to_ftp", "Upload last backup to FTP"
|
128
|
+
def upload_to_ftp
|
129
|
+
invoke :download, [], application: options[:application], addon: options[:addon]
|
130
|
+
configuration = Configuration.new
|
131
|
+
opts = {
|
132
|
+
webhooks: configuration.config[:webhooks]
|
133
|
+
}
|
134
|
+
configuration.for_each_addons do |application, addon|
|
135
|
+
step = 1
|
136
|
+
sftp_config = addon.sftp_config
|
137
|
+
path = ("#{addon.config[:path]}" || "backups/#{addon.addon_provider[:id]}") + "/#{Time.now.strftime("%Y%m%d")}.tar.gz"
|
138
|
+
next unless File.exists?(path)
|
139
|
+
puts "** Upload backup for #{application.name} **"
|
140
|
+
|
141
|
+
sftp = ScalingoBackupsManager::SftpTools.new(sftp_config[:auth])
|
142
|
+
|
143
|
+
folders = [
|
144
|
+
sftp_config.dig(:auth, :dir),
|
145
|
+
sftp_config.dig(:dir) || application.name,
|
146
|
+
addon.addon_provider[:id]
|
147
|
+
]
|
148
|
+
|
149
|
+
if sftp_config[:retention].blank?
|
150
|
+
remote_path = "/" + [folders].delete_if(&:blank?).join("/")
|
151
|
+
sftp.mkdir!(remote_path)
|
152
|
+
sftp.upload_file(path, remote_path, options: opts)
|
153
|
+
next
|
154
|
+
end
|
155
|
+
|
156
|
+
sftp_config[:retention].each do |k, retention_config|
|
157
|
+
retention_folders = folders.dup
|
158
|
+
retention_folders << sftp_config.dig(:retention, k, :dir)
|
159
|
+
remote_path = "/" + retention_folders.delete_if(&:blank?).join("/")
|
160
|
+
puts "#{step} - Creating remote directory at #{remote_path}"
|
161
|
+
step += 1
|
162
|
+
sftp.mkdir!(remote_path)
|
163
|
+
case k
|
164
|
+
when "daily"
|
165
|
+
sftp.upload_file(path, remote_path, options: opts)
|
166
|
+
files = sftp.list_files(remote_path)
|
167
|
+
puts "#{step} - Checking daily backups"
|
168
|
+
step += 1
|
169
|
+
if files.size > retention_config[:ttl]
|
170
|
+
files_to_remove = files.sort_by(&:name).shift(files.size - retention_config[:ttl])
|
171
|
+
puts "#{step} - Removing #{files_to_remove.size} backups because of ttl configuration"
|
172
|
+
files_to_remove.each do |file|
|
173
|
+
puts "Removing file #{remote_path + "/" + file.name}"
|
174
|
+
sftp.remove!(remote_path + "/" + file.name)
|
175
|
+
end
|
176
|
+
end
|
177
|
+
when "monthly"
|
178
|
+
next unless Date.today.day == 1
|
179
|
+
sftp.upload_file(path, remote_path, options: opts)
|
180
|
+
puts "#{step} - Checking monthly backups"
|
181
|
+
step += 1
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
end
|
186
|
+
FileUtils.rm_r 'backups/'
|
187
|
+
end
|
188
|
+
|
125
189
|
end
|
126
190
|
|
127
191
|
private
|
@@ -16,7 +16,6 @@ module ScalingoBackupsManager
|
|
16
16
|
def self.create_file
|
17
17
|
File.open(FILE_NAME, 'w+') do |f|
|
18
18
|
f.flock(File::LOCK_EX)
|
19
|
-
content = f.read
|
20
19
|
f.rewind
|
21
20
|
f.write({ region: 'osc-fr-1', apps: []}.to_yaml)
|
22
21
|
f.flush
|
@@ -34,7 +33,7 @@ module ScalingoBackupsManager
|
|
34
33
|
File.open(FILE_NAME, "w+") do |f|
|
35
34
|
f.flock(File::LOCK_EX)
|
36
35
|
f.truncate(0)
|
37
|
-
f.write(config.to_yaml)
|
36
|
+
f.write(config.to_hash.to_yaml)
|
38
37
|
f.flush
|
39
38
|
end
|
40
39
|
end
|
@@ -45,7 +44,7 @@ module ScalingoBackupsManager
|
|
45
44
|
puts "Configuration file does not exist"
|
46
45
|
return
|
47
46
|
end
|
48
|
-
@config = YAML.load(@file.read) || { apps: [] }
|
47
|
+
@config = (YAML.load(@file.read) || { apps: [] }).with_indifferent_access
|
49
48
|
end
|
50
49
|
|
51
50
|
def add_addon_to_app(application, addon)
|
@@ -53,16 +52,21 @@ module ScalingoBackupsManager
|
|
53
52
|
self.class.write_config(@config)
|
54
53
|
end
|
55
54
|
|
56
|
-
def for_each_addons(application_uid, addon_uid)
|
55
|
+
def for_each_addons(application_uid = nil, addon_uid = nil)
|
56
|
+
client_sftp_config = @config[:sftp] || {}
|
57
|
+
|
57
58
|
@config[:apps].each do |application_name, application_config|
|
58
59
|
next if application_uid && application_uid != application_name.to_s
|
59
60
|
next unless application_config[:id]
|
60
61
|
application = ScalingoBackupsManager::Application.find(application_config[:id])
|
61
62
|
next unless application_config[:addons] && application_config[:addons].size > 0
|
63
|
+
|
64
|
+
app_sftp_config = (application_config[:sftp] || {}).reverse_merge(client_sftp_config)
|
65
|
+
|
62
66
|
application_config[:addons].each do |addon_name, addon_config|
|
63
67
|
next if addon_uid && addon_uid != addon_name.to_s
|
64
68
|
next unless addon_config[:id]
|
65
|
-
addon = ScalingoBackupsManager::Addon.find(application, addon_config[:id], config: addon_config)
|
69
|
+
addon = ScalingoBackupsManager::Addon.find(application, addon_config[:id], config: addon_config, sftp_config: (addon_config[:sftp] || {}).reverse_merge(app_sftp_config))
|
66
70
|
yield(application, addon) if block_given?
|
67
71
|
end
|
68
72
|
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'httparty'
|
2
|
+
module ScalingoBackupsManager
|
3
|
+
|
4
|
+
class Notification
|
5
|
+
|
6
|
+
def self.send_slack_notification(hook_url, message)
|
7
|
+
HTTParty.post(
|
8
|
+
hook_url,
|
9
|
+
body: {
|
10
|
+
message: message
|
11
|
+
}.to_json,
|
12
|
+
headers: { 'Content-Type' => 'application/json' }
|
13
|
+
)
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.send_discord_notification(hook_url, message)
|
17
|
+
payload = {
|
18
|
+
user: 'Scalingo backups manager',
|
19
|
+
content: message
|
20
|
+
}.to_json
|
21
|
+
HTTParty.post(
|
22
|
+
hook_url,
|
23
|
+
body: payload,
|
24
|
+
headers: { 'Content-Type': 'application/json' }
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'scalingo_backups_manager/notification'
|
2
|
+
require 'net/sftp'
|
3
|
+
|
4
|
+
module ScalingoBackupsManager
|
5
|
+
class SftpTools
|
6
|
+
attr_accessor :ftp_host
|
7
|
+
|
8
|
+
def initialize(ftp_host)
|
9
|
+
@ftp_host = ftp_host
|
10
|
+
end
|
11
|
+
|
12
|
+
def start
|
13
|
+
Net::SFTP.start(@ftp_host[:host], @ftp_host[:user], password: @ftp_host[:password], port: @ftp_host[:port]) do |sftp|
|
14
|
+
yield(sftp) if block_given?
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def list_files(path)
|
19
|
+
files = []
|
20
|
+
start do |sftp|
|
21
|
+
sftp.dir.glob("#{path}", "*.tar.gz").each do |file|
|
22
|
+
files << file
|
23
|
+
end
|
24
|
+
end
|
25
|
+
files
|
26
|
+
end
|
27
|
+
|
28
|
+
def remove!(path)
|
29
|
+
start do |sftp|
|
30
|
+
sftp.remove!(path)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def mkdir!(path)
|
35
|
+
start do |sftp|
|
36
|
+
folder_tree = []
|
37
|
+
path.split("/").each do |folder_name|
|
38
|
+
next if folder_name.blank?
|
39
|
+
|
40
|
+
folder_tree << folder_name
|
41
|
+
begin
|
42
|
+
sftp.mkdir!(folder_tree.join("/"))
|
43
|
+
rescue
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def upload_file(filepath, remote_dir, options: {})
|
50
|
+
filename = filepath.split('/').last
|
51
|
+
start do |sftp|
|
52
|
+
begin
|
53
|
+
sftp.upload!(filepath, "#{remote_dir}/#{filename}")
|
54
|
+
rescue
|
55
|
+
if options.dig(:webhooks, :slack_webhook_url)
|
56
|
+
ScalingoBackupsManager::Notification.send_slack_notification(options.dig(:webhooks, :slack_webhook_url), "An error has occured while uploading backup, see the logs for more information")
|
57
|
+
end
|
58
|
+
if options.dig(:webhooks, :discord_webhook_url)
|
59
|
+
ScalingoBackupsManager::Notification.send_discord_notification(options.dig(:webhooks, :discord_webhook_url), "An error has occured while uploading backup, see the logs for more information")
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
@@ -18,6 +18,7 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.add_dependency "thor", '~> 1.1'
|
19
19
|
spec.add_dependency 'httparty', "~> 0.18"
|
20
20
|
spec.add_dependency 'scalingo', '~> 3.0'
|
21
|
+
spec.add_dependency 'net-sftp', '~> 3.0.0'
|
21
22
|
|
22
23
|
# Specify which files should be added to the gem when it is released.
|
23
24
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: scalingo_backups_manager
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kevin Clercin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-06-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '3.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: net-sftp
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 3.0.0
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 3.0.0
|
55
69
|
description:
|
56
70
|
email:
|
57
71
|
- kevin@9troisquarts.com
|
@@ -78,9 +92,11 @@ files:
|
|
78
92
|
- lib/scalingo_backups_manager/backup.rb
|
79
93
|
- lib/scalingo_backups_manager/cli.rb
|
80
94
|
- lib/scalingo_backups_manager/configuration.rb
|
95
|
+
- lib/scalingo_backups_manager/notification.rb
|
81
96
|
- lib/scalingo_backups_manager/restore/mongodb.rb
|
82
97
|
- lib/scalingo_backups_manager/restore/mysql.rb
|
83
98
|
- lib/scalingo_backups_manager/restore/postgres.rb
|
99
|
+
- lib/scalingo_backups_manager/sftp_tools.rb
|
84
100
|
- lib/scalingo_backups_manager/version.rb
|
85
101
|
- scalingo_backups_manager.gemspec
|
86
102
|
homepage: https://github.com/9troisquarts/scalingo_backups_manager
|