capistrano-ops 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/capistrano/ops/backup/s3.rb +26 -19
- data/lib/capistrano/ops/capistrano/v3/tasks/backup/database/create.rake +1 -1
- data/lib/capistrano/ops/capistrano/v3/tasks/backup/storage/create.rake +1 -1
- data/lib/capistrano/ops/tasks/pg/dump.rake +14 -46
- data/lib/capistrano/ops/tasks/pg/postgres_helper.rb +67 -0
- data/lib/capistrano/ops/tasks/pg/remove_old_dumps.rake +7 -6
- data/lib/capistrano/ops/tasks/storage/backup.rake +10 -18
- data/lib/capistrano/ops/tasks/storage/remove_old_backups.rake +4 -3
- data/lib/capistrano/ops/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1ddab185886973237cac8aab6671b9bb580d5496f72e37f0678b6b0dd794c78a
|
4
|
+
data.tar.gz: 4c12ffe04071978d364f6a02b5d0cfd3a0e651a8669b86c00e2445c1830bc74b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 36dc0e5814ab7ff1b7370b366cf8dce0f8c4092f300319c7d2f936c554455ad7bd1edb5e519c80157515f0590d55ca4f995f9dbbadcd72d7803040e7c4ffb5bd
|
7
|
+
data.tar.gz: e2a9c85b8a361cca26a84bdade6500d31f8ab63e3f792e002dd7db5b3949779eb9fbfe5fccad7496fc4baff6baa7dbb3c6921e933ca7dd11a3f3ba61ca83bec0
|
@@ -3,7 +3,7 @@
|
|
3
3
|
module Backup
|
4
4
|
require 'aws-sdk-s3'
|
5
5
|
class S3
|
6
|
-
attr_accessor :endpoint, :region, :access_key_id, :secret_access_key, :
|
6
|
+
attr_accessor :endpoint, :region, :access_key_id, :secret_access_key, :s3_resource
|
7
7
|
|
8
8
|
def initialize(endpoint: ENV['S3_BACKUP_ENDPOINT'], region: ENV['S3_BACKUP_REGION'], access_key_id: ENV['S3_BACKUP_KEY'],
|
9
9
|
secret_access_key: ENV['S3_BACKUP_SECRET'])
|
@@ -11,42 +11,49 @@ module Backup
|
|
11
11
|
self.region = region
|
12
12
|
self.access_key_id = access_key_id
|
13
13
|
self.secret_access_key = secret_access_key
|
14
|
-
|
14
|
+
configuration = {
|
15
15
|
region: region,
|
16
16
|
access_key_id: access_key_id,
|
17
17
|
secret_access_key: secret_access_key
|
18
18
|
}
|
19
|
-
|
20
|
-
self.
|
19
|
+
configuration[:endpoint] = endpoint unless endpoint.nil?
|
20
|
+
self.s3_resource = Aws::S3::Resource.new(configuration)
|
21
21
|
end
|
22
22
|
|
23
23
|
def upload(backup_file, key)
|
24
24
|
begin
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
body: File.open(backup_file)
|
29
|
-
)
|
30
|
-
rescue StandardError => e
|
31
|
-
puts "Error uploading backup to S3: #{e.message}"
|
25
|
+
s3_resource.bucket(ENV['S3_BACKUP_BUCKET']).object(key).upload_file(backup_file)
|
26
|
+
rescue Backup::Error => e
|
27
|
+
puts "Upload failed: #{e.message}"
|
32
28
|
raise e
|
33
29
|
end
|
34
30
|
'File uploaded to S3'
|
35
31
|
end
|
36
32
|
|
37
33
|
def remove_old_backups(basename, keep: 5)
|
38
|
-
|
39
|
-
|
34
|
+
all_items = s3_resource.bucket(ENV['S3_BACKUP_BUCKET']).objects(prefix: basename).map do |item|
|
35
|
+
{ key: item.key, last_modified: item.last_modified }
|
36
|
+
end
|
37
|
+
|
38
|
+
all_items = all_items.sort_by { |hsh| hsh[:last_modified] }.reverse
|
39
|
+
|
40
40
|
count = all_items.count
|
41
|
+
|
41
42
|
if count <= keep
|
42
|
-
|
43
|
-
|
43
|
+
puts 'Nothing to remove'
|
44
|
+
exit(0)
|
44
45
|
end
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
46
|
+
|
47
|
+
delete_items = all_items.slice(keep..-1)
|
48
|
+
|
49
|
+
delete_items.each do |item_obj|
|
50
|
+
puts "Removing #{item_obj[:key]} from S3"
|
51
|
+
s3_resource.bucket(ENV['S3_BACKUP_BUCKET']).object(item_obj[:key]).delete
|
49
52
|
end
|
53
|
+
puts 'Old backups removed from S3'
|
54
|
+
rescue Backup::Error => e
|
55
|
+
puts "Remove failed: #{e.message}"
|
56
|
+
raise e
|
50
57
|
end
|
51
58
|
end
|
52
59
|
end
|
@@ -1,27 +1,25 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
require_relative './postgres_helper'
|
4
4
|
namespace :pg do
|
5
|
-
|
6
|
-
|
7
|
-
@password = Rails.configuration.database_configuration[Rails.env]['password']
|
8
|
-
@hostname = Rails.configuration.database_configuration[Rails.env]['host']
|
9
|
-
@portnumber = Rails.configuration.database_configuration[Rails.env]['port']
|
10
|
-
@backup_path = Rails.root.join(Rails.env.development? ? 'tmp/backups' : '../../shared/backups').to_s
|
11
|
-
backups_enabled = Rails.env.production? || ENV['BACKUPS_ENABLED'] == 'true'
|
12
|
-
external_backup = Rails.env.production? || ENV['EXTERNAL_BACKUP_ENABLED'] == 'true'
|
5
|
+
include PostgresHelper
|
6
|
+
|
13
7
|
task :dump do
|
8
|
+
backup_path = configuration[:backup_path]
|
9
|
+
backups_enabled = configuration[:backups_enabled]
|
10
|
+
external_backup = configuration[:external_backup]
|
11
|
+
|
14
12
|
unless backups_enabled
|
15
13
|
puts 'dump: Backups are disabled'
|
16
14
|
exit(0)
|
17
15
|
end
|
18
16
|
|
19
17
|
notification = Notification::Api.new
|
20
|
-
commandlist = dump_cmd
|
18
|
+
commandlist = dump_cmd(configuration)
|
21
19
|
|
22
|
-
system "mkdir -p #{
|
20
|
+
system "mkdir -p #{backup_path}" unless Dir.exist?(backup_path)
|
23
21
|
|
24
|
-
result = system(commandlist
|
22
|
+
result = system(commandlist)
|
25
23
|
|
26
24
|
if ENV['BACKUP_PROVIDER'].present? && external_backup && result
|
27
25
|
puts "Uploading #{@filename} to #{ENV['BACKUP_PROVIDER']}..."
|
@@ -33,40 +31,10 @@ namespace :pg do
|
|
33
31
|
puts "#{@filename} upload failed: #{e.message}"
|
34
32
|
end
|
35
33
|
end
|
36
|
-
notification.send_backup_notification(result, title, content(result
|
37
|
-
|
38
|
-
|
39
|
-
end
|
40
|
-
|
41
|
-
def title
|
42
|
-
ENV['DEFAULT_URL'] || "#{Rails.env} Backup"
|
43
|
-
end
|
44
|
-
|
45
|
-
def content(result)
|
46
|
-
messages = []
|
47
|
-
if result
|
48
|
-
messages << "Backup of #{@database} successfully finished at #{Time.now}"
|
49
|
-
messages << "Backup path:\`#{@backup_path}/#{@filename}\`"
|
50
|
-
else
|
51
|
-
messages << "Backup of #{@database} failed at #{Time.now}"
|
52
|
-
end
|
53
|
-
messages.join("\n")
|
54
|
-
end
|
34
|
+
notification.send_backup_notification(result, title, content(result, { database: @database, backup_path: @backup_path, filename: @filename }),
|
35
|
+
{ date: @date, backup_path: @backup_path, database: @database })
|
36
|
+
puts result ? "Backup created: #{@backup_path}/#{@filename} (#{size_str(File.size("#{@backup_path}/#{@filename}"))})" : 'Backup failed removing dump file'
|
55
37
|
|
56
|
-
|
57
|
-
@date = Time.now.to_i
|
58
|
-
options = []
|
59
|
-
options << " -d #{@database}" if @database.present?
|
60
|
-
options << " -U #{@username}" if @username.present?
|
61
|
-
options << " -h #{@hostname}" if @hostname.present?
|
62
|
-
options << " -p #{@portnumber}" if @portnumber.present?
|
63
|
-
|
64
|
-
@filename = "#{@database}_#{@date}.dump"
|
65
|
-
|
66
|
-
commandlist = []
|
67
|
-
commandlist << "export PGPASSWORD='#{@password}'"
|
68
|
-
commandlist << "cd #{@backup_path}"
|
69
|
-
commandlist << "pg_dump -Fc #{options.join('')} > #{@filename}"
|
38
|
+
system "rm #{@backup_path}/#{@filename}" unless result
|
70
39
|
end
|
71
40
|
end
|
72
|
-
# rubocop:enable Metrics/BlockLength
|
@@ -0,0 +1,67 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module PostgresHelper
|
4
|
+
def configuration
|
5
|
+
@configuration ||=
|
6
|
+
{
|
7
|
+
database: Rails.configuration.database_configuration[Rails.env]['database'],
|
8
|
+
username: Rails.configuration.database_configuration[Rails.env]['username'],
|
9
|
+
password: Rails.configuration.database_configuration[Rails.env]['password'],
|
10
|
+
hostname: Rails.configuration.database_configuration[Rails.env]['host'],
|
11
|
+
portnumber: Rails.configuration.database_configuration[Rails.env]['port'],
|
12
|
+
backup_path: Rails.root.join(Rails.env.development? ? 'tmp/backups' : '../../shared/backups').to_s,
|
13
|
+
backups_enabled: Rails.env.production? || ENV['BACKUPS_ENABLED'] == 'true',
|
14
|
+
external_backup: Rails.env.production? || ENV['EXTERNAL_BACKUP_ENABLED'] == 'true'
|
15
|
+
}
|
16
|
+
end
|
17
|
+
|
18
|
+
def title
|
19
|
+
ENV['DEFAULT_URL'] || "#{Rails.env} Backup"
|
20
|
+
end
|
21
|
+
|
22
|
+
def content(result, settings = {})
|
23
|
+
@database = settings[:database]
|
24
|
+
@backup_path = settings[:backup_path]
|
25
|
+
@filename = settings[:filename]
|
26
|
+
|
27
|
+
messages = []
|
28
|
+
if result
|
29
|
+
messages << "Backup of #{@database} successfully finished at #{Time.now}"
|
30
|
+
messages << "Backup path:\`#{@backup_path}/#{@filename}\`"
|
31
|
+
else
|
32
|
+
messages << "Backup of #{@database} failed at #{Time.now}"
|
33
|
+
end
|
34
|
+
messages.join("\n")
|
35
|
+
end
|
36
|
+
|
37
|
+
def dump_cmd(settings = {})
|
38
|
+
@hostname = settings[:hostname]
|
39
|
+
@database = settings[:database]
|
40
|
+
@username = settings[:username]
|
41
|
+
@password = settings[:password]
|
42
|
+
@portnumber = settings[:portnumber]
|
43
|
+
@backup_path = settings[:backup_path]
|
44
|
+
|
45
|
+
@date = Time.now.to_i
|
46
|
+
options = []
|
47
|
+
options << " -d #{@database}" if @database.present?
|
48
|
+
options << " -U #{@username}" if @username.present?
|
49
|
+
options << " -h #{@hostname}" if @hostname.present?
|
50
|
+
options << " -p #{@portnumber}" if @portnumber.present?
|
51
|
+
|
52
|
+
@filename = "#{@database}_#{@date}.dump"
|
53
|
+
|
54
|
+
commandlist = []
|
55
|
+
commandlist << "export PGPASSWORD='#{@password}'"
|
56
|
+
commandlist << "cd #{@backup_path}"
|
57
|
+
commandlist << "pg_dump -Fc #{options.join('')} > #{@filename}"
|
58
|
+
commandlist.join(' && ')
|
59
|
+
end
|
60
|
+
|
61
|
+
def size_str(size)
|
62
|
+
units = %w[B KB MB GB TB]
|
63
|
+
e = (Math.log(size) / Math.log(1024)).floor
|
64
|
+
s = format('%.2f', size.to_f / 1024**e)
|
65
|
+
s.sub(/\.?0*$/, units[e])
|
66
|
+
end
|
67
|
+
end
|
@@ -4,8 +4,8 @@ require 'rake'
|
|
4
4
|
namespace :pg do
|
5
5
|
@backup_path = Rails.root.join(Rails.env.development? ? 'tmp/backups' : '../../shared/backups').to_s
|
6
6
|
@database = Rails.configuration.database_configuration[Rails.env]['database']
|
7
|
-
@env_local_no = ENV['NUMBER_OF_LOCAL_BACKUPS']
|
8
|
-
@env_external_no = ENV['NUMBER_OF_EXTERNAL_BACKUPS']
|
7
|
+
@env_local_no = ENV['NUMBER_OF_LOCAL_BACKUPS'].present? ? ENV['NUMBER_OF_LOCAL_BACKUPS'] : nil
|
8
|
+
@env_external_no = ENV['NUMBER_OF_EXTERNAL_BACKUPS'].present? ? ENV['NUMBER_OF_EXTERNAL_BACKUPS'] : nil
|
9
9
|
@total_local_backups_no = (@env_local_no || ENV['NUMBER_OF_BACKUPS'] || 7).to_i
|
10
10
|
@total_external_backups_no = (@env_external_no || ENV['NUMBER_OF_BACKUPS'] || 7).to_i
|
11
11
|
backups_enabled = Rails.env.production? || ENV['BACKUPS_ENABLED'] == 'true'
|
@@ -19,7 +19,7 @@ namespace :pg do
|
|
19
19
|
exit(0)
|
20
20
|
end
|
21
21
|
unless @total_local_backups_no.positive?
|
22
|
-
puts "remove_old_dumps: No local cleanup because option '#{if @env_local_no
|
22
|
+
puts "remove_old_dumps: No local cleanup because option '#{if @env_local_no.present?
|
23
23
|
'NUMBER_OF_LOCAL_BACKUPS='
|
24
24
|
else
|
25
25
|
'NUMBER_OF_BACKUPS='
|
@@ -35,11 +35,12 @@ namespace :pg do
|
|
35
35
|
'xargs rm -rf'
|
36
36
|
]
|
37
37
|
|
38
|
-
system(commandlist.join(' | '))
|
38
|
+
result = system(commandlist.join(' | ')) if @total_local_backups_no.positive?
|
39
|
+
puts 'remove_old_dumps: local cleanup finished' if result
|
39
40
|
|
40
41
|
if ENV['BACKUP_PROVIDER'].present? && external_backup
|
41
42
|
unless @total_external_backups_no.positive?
|
42
|
-
puts "remove_old_dumps: No external cleanup because option '#{if @env_external_no
|
43
|
+
puts "remove_old_dumps: No external cleanup because option '#{if @env_external_no.present?
|
43
44
|
'NUMBER_OF_EXTERNAL_BACKUPS='
|
44
45
|
else
|
45
46
|
'NUMBER_OF_BACKUPS='
|
@@ -48,7 +49,7 @@ namespace :pg do
|
|
48
49
|
end
|
49
50
|
provider = Backup::Api.new
|
50
51
|
begin
|
51
|
-
result = provider.remove_old_backups(@database, @total_external_backups_no)
|
52
|
+
result = provider.remove_old_backups("#{@database}_", @total_external_backups_no)
|
52
53
|
rescue StandardError => e
|
53
54
|
puts "remove_old_dumps failed: #{e.message}"
|
54
55
|
end
|
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
# rubocop:disable Metrics/BlockLength
|
4
3
|
namespace :storage do
|
5
4
|
@backup_path = Rails.root.join(Rails.env.development? ? 'tmp/backups' : '../../shared/backups').to_s
|
6
5
|
@storage_path = Rails.root.join(Rails.env.development? ? 'storage' : '../../shared/storage').to_s
|
@@ -18,11 +17,11 @@ namespace :storage do
|
|
18
17
|
date = Time.now.to_i
|
19
18
|
@filename = "storage_#{date}.tar.gz"
|
20
19
|
FileUtils.mkdir_p(@backup_path) unless Dir.exist?(@backup_path)
|
21
|
-
|
22
|
-
FileUtils.rm_rf("#{@backup_path}/#{filename}") unless
|
23
|
-
puts
|
20
|
+
response = system "tar -zcf #{@backup_path}/#{@filename} -C #{@storage_path} ."
|
21
|
+
FileUtils.rm_rf("#{@backup_path}/#{filename}") unless response
|
22
|
+
puts response ? "Backup created: #{@backup_path}/#{@filename} (#{size_str(File.size("#{@backup_path}/#{@filename}"))})" : 'Backup failed removing dump file'
|
24
23
|
|
25
|
-
if ENV['BACKUP_PROVIDER'].present? && external_backup &&
|
24
|
+
if ENV['BACKUP_PROVIDER'].present? && external_backup && response
|
26
25
|
puts "Uploading #{@filename} to #{ENV['BACKUP_PROVIDER']}..."
|
27
26
|
provider = Backup::Api.new
|
28
27
|
begin
|
@@ -32,14 +31,14 @@ namespace :storage do
|
|
32
31
|
puts "#{@filename} upload failed: #{e.message}"
|
33
32
|
end
|
34
33
|
end
|
35
|
-
notification.send_backup_notification(
|
34
|
+
notification.send_backup_notification(response, title, message(response), { date: date, backup_path: @backup_path, database: 'storage' })
|
36
35
|
end
|
37
36
|
|
38
37
|
def title
|
39
38
|
ENV['DEFAULT_URL'] || "#{Rails.env} Backup"
|
40
39
|
end
|
41
40
|
|
42
|
-
def message(result)
|
41
|
+
def message(result=false)
|
43
42
|
messages = []
|
44
43
|
if result
|
45
44
|
messages << "Backup of storage folder successfully finished at #{Time.now}"
|
@@ -51,16 +50,9 @@ namespace :storage do
|
|
51
50
|
end
|
52
51
|
|
53
52
|
def size_str(size)
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
"#{size / 1024} KB"
|
59
|
-
when 1024 * 1024..1024 * 1024 * 1024
|
60
|
-
"#{size / 1024 / 1024} MB"
|
61
|
-
when 1024 * 1024 * 1024..1024 * 1024 * 1024 * 1024
|
62
|
-
"#{size / 1024 / 1024 / 1024} GB"
|
63
|
-
end
|
53
|
+
units = %w[B KB MB GB TB]
|
54
|
+
e = (Math.log(size) / Math.log(1024)).floor
|
55
|
+
s = format('%.2f', size.to_f / 1024**e)
|
56
|
+
s.sub(/\.?0*$/, units[e])
|
64
57
|
end
|
65
58
|
end
|
66
|
-
# rubocop:enable Metrics/BlockLength
|
@@ -6,8 +6,8 @@ namespace :storage do
|
|
6
6
|
backups_enabled = Rails.env.production? || ENV['BACKUPS_ENABLED'] == 'true'
|
7
7
|
external_backup = Rails.env.production? || ENV['EXTERNAL_BACKUP_ENABLED'] == 'true'
|
8
8
|
|
9
|
-
@env_local_no = ENV['NUMBER_OF_LOCAL_BACKUPS']
|
10
|
-
@env_external_no = ENV['NUMBER_OF_EXTERNAL_BACKUPS']
|
9
|
+
@env_local_no = ENV['NUMBER_OF_LOCAL_BACKUPS'].present? ? ENV['NUMBER_OF_LOCAL_BACKUPS'] : nil
|
10
|
+
@env_external_no = ENV['NUMBER_OF_EXTERNAL_BACKUPS'].present? ? ENV['NUMBER_OF_EXTERNAL_BACKUPS'] : nil
|
11
11
|
@total_local_backups_no = (@env_local_no || ENV['NUMBER_OF_BACKUPS'] || 7).to_i
|
12
12
|
@total_external_backups_no = (@env_external_no || ENV['NUMBER_OF_BACKUPS'] || 7).to_i
|
13
13
|
desc 'remove old storage backups'
|
@@ -35,7 +35,8 @@ namespace :storage do
|
|
35
35
|
'xargs rm -rf'
|
36
36
|
]
|
37
37
|
|
38
|
-
system(commandlist.join(' | '))
|
38
|
+
result = system(commandlist.join(' | ')) if @total_local_backups_no.positive?
|
39
|
+
puts 'remove_old_backups: local cleanup finished' if result
|
39
40
|
|
40
41
|
if ENV['BACKUP_PROVIDER'].present? && external_backup
|
41
42
|
unless @total_external_backups_no.positive?
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: capistrano-ops
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Florian Crusius
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-08-
|
11
|
+
date: 2023-08-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk-s3
|
@@ -135,6 +135,7 @@ files:
|
|
135
135
|
- lib/capistrano/ops/notification/webhook.rb
|
136
136
|
- lib/capistrano/ops/railtie.rb
|
137
137
|
- lib/capistrano/ops/tasks/pg/dump.rake
|
138
|
+
- lib/capistrano/ops/tasks/pg/postgres_helper.rb
|
138
139
|
- lib/capistrano/ops/tasks/pg/remove_old_dumps.rake
|
139
140
|
- lib/capistrano/ops/tasks/storage/backup.rake
|
140
141
|
- lib/capistrano/ops/tasks/storage/remove_old_backups.rake
|