backup 4.4.1 → 5.0.0.beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/LICENSE +19 -0
- data/README.md +1 -1
- data/lib/backup.rb +74 -78
- data/lib/backup/archive.rb +31 -32
- data/lib/backup/binder.rb +2 -6
- data/lib/backup/cleaner.rb +14 -18
- data/lib/backup/cli.rb +104 -108
- data/lib/backup/cloud_io/base.rb +4 -7
- data/lib/backup/cloud_io/cloud_files.rb +60 -62
- data/lib/backup/cloud_io/s3.rb +69 -76
- data/lib/backup/compressor/base.rb +4 -7
- data/lib/backup/compressor/bzip2.rb +3 -7
- data/lib/backup/compressor/custom.rb +2 -6
- data/lib/backup/compressor/gzip.rb +16 -17
- data/lib/backup/config.rb +17 -18
- data/lib/backup/config/dsl.rb +16 -17
- data/lib/backup/config/helpers.rb +10 -16
- data/lib/backup/database/base.rb +22 -21
- data/lib/backup/database/mongodb.rb +36 -37
- data/lib/backup/database/mysql.rb +40 -41
- data/lib/backup/database/openldap.rb +8 -10
- data/lib/backup/database/postgresql.rb +29 -30
- data/lib/backup/database/redis.rb +27 -30
- data/lib/backup/database/riak.rb +15 -18
- data/lib/backup/database/sqlite.rb +4 -6
- data/lib/backup/encryptor/base.rb +2 -4
- data/lib/backup/encryptor/gpg.rb +49 -59
- data/lib/backup/encryptor/open_ssl.rb +11 -14
- data/lib/backup/errors.rb +7 -12
- data/lib/backup/logger.rb +16 -18
- data/lib/backup/logger/console.rb +5 -8
- data/lib/backup/logger/fog_adapter.rb +2 -6
- data/lib/backup/logger/logfile.rb +10 -12
- data/lib/backup/logger/syslog.rb +2 -4
- data/lib/backup/model.rb +75 -40
- data/lib/backup/notifier/base.rb +24 -26
- data/lib/backup/notifier/campfire.rb +9 -11
- data/lib/backup/notifier/command.rb +0 -3
- data/lib/backup/notifier/datadog.rb +9 -12
- data/lib/backup/notifier/flowdock.rb +13 -17
- data/lib/backup/notifier/hipchat.rb +11 -13
- data/lib/backup/notifier/http_post.rb +11 -14
- data/lib/backup/notifier/mail.rb +44 -47
- data/lib/backup/notifier/nagios.rb +5 -9
- data/lib/backup/notifier/pagerduty.rb +10 -12
- data/lib/backup/notifier/prowl.rb +15 -15
- data/lib/backup/notifier/pushover.rb +7 -10
- data/lib/backup/notifier/ses.rb +34 -16
- data/lib/backup/notifier/slack.rb +39 -40
- data/lib/backup/notifier/twitter.rb +2 -5
- data/lib/backup/notifier/zabbix.rb +11 -14
- data/lib/backup/package.rb +5 -9
- data/lib/backup/packager.rb +16 -17
- data/lib/backup/pipeline.rb +17 -21
- data/lib/backup/splitter.rb +8 -11
- data/lib/backup/storage/base.rb +5 -8
- data/lib/backup/storage/cloud_files.rb +21 -23
- data/lib/backup/storage/cycler.rb +10 -15
- data/lib/backup/storage/dropbox.rb +15 -21
- data/lib/backup/storage/ftp.rb +8 -10
- data/lib/backup/storage/local.rb +5 -8
- data/lib/backup/storage/qiniu.rb +8 -8
- data/lib/backup/storage/rsync.rb +24 -26
- data/lib/backup/storage/s3.rb +27 -28
- data/lib/backup/storage/scp.rb +10 -12
- data/lib/backup/storage/sftp.rb +10 -12
- data/lib/backup/syncer/base.rb +5 -8
- data/lib/backup/syncer/cloud/base.rb +27 -30
- data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
- data/lib/backup/syncer/cloud/local_file.rb +5 -8
- data/lib/backup/syncer/cloud/s3.rb +23 -24
- data/lib/backup/syncer/rsync/base.rb +6 -10
- data/lib/backup/syncer/rsync/local.rb +1 -5
- data/lib/backup/syncer/rsync/pull.rb +6 -10
- data/lib/backup/syncer/rsync/push.rb +18 -22
- data/lib/backup/template.rb +9 -14
- data/lib/backup/utilities.rb +82 -69
- data/lib/backup/version.rb +1 -3
- metadata +100 -660
@@ -1,9 +1,6 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Notifier
|
5
3
|
class Nagios < Base
|
6
|
-
|
7
4
|
##
|
8
5
|
# Host of Nagios server to notify on backup completion.
|
9
6
|
attr_accessor :nagios_host
|
@@ -30,8 +27,8 @@ module Backup
|
|
30
27
|
|
31
28
|
@nagios_host ||= Config.hostname
|
32
29
|
@nagios_port ||= 5667
|
33
|
-
@send_nsca_cfg||= "/etc/nagios/send_nsca.cfg"
|
34
|
-
@service_name ||= "Backup #{
|
30
|
+
@send_nsca_cfg ||= "/etc/nagios/send_nsca.cfg"
|
31
|
+
@service_name ||= "Backup #{model.trigger}"
|
35
32
|
@service_host ||= Config.hostname
|
36
33
|
end
|
37
34
|
|
@@ -55,15 +52,14 @@ module Backup
|
|
55
52
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
56
53
|
#
|
57
54
|
def notify!(status)
|
58
|
-
send_message(message.call(model, :
|
55
|
+
send_message(message.call(model, status: status_data_for(status)))
|
59
56
|
end
|
60
57
|
|
61
58
|
def send_message(message)
|
62
|
-
cmd = "#{
|
59
|
+
cmd = "#{utility(:send_nsca)} -H '#{nagios_host}' -p '#{nagios_port}' -c '#{send_nsca_cfg}'"
|
63
60
|
msg = [service_host, service_name, model.exit_status, message].join("\t")
|
64
|
-
run("echo '#{
|
61
|
+
run("echo '#{msg}' | #{cmd}")
|
65
62
|
end
|
66
|
-
|
67
63
|
end
|
68
64
|
end
|
69
65
|
end
|
@@ -1,10 +1,8 @@
|
|
1
|
-
|
2
|
-
require 'pagerduty'
|
1
|
+
require "pagerduty"
|
3
2
|
|
4
3
|
module Backup
|
5
4
|
module Notifier
|
6
5
|
class PagerDuty < Base
|
7
|
-
|
8
6
|
##
|
9
7
|
# PagerDuty Service API Key. Should be a 32 character hex string.
|
10
8
|
attr_accessor :service_key
|
@@ -46,15 +44,15 @@ module Backup
|
|
46
44
|
incident_description = "Backup - #{model.label}"
|
47
45
|
incident_key = "backup/#{model.trigger}"
|
48
46
|
incident_details = {
|
49
|
-
:
|
50
|
-
:
|
51
|
-
:
|
52
|
-
:
|
53
|
-
:
|
54
|
-
:
|
55
|
-
:
|
56
|
-
:
|
57
|
-
:
|
47
|
+
incident_key: incident_key,
|
48
|
+
details: {
|
49
|
+
trigger: model.trigger,
|
50
|
+
label: model.label,
|
51
|
+
started_at: model.started_at,
|
52
|
+
finished_at: model.finished_at,
|
53
|
+
duration: model.duration,
|
54
|
+
status: status,
|
55
|
+
exception: model.exception
|
58
56
|
}
|
59
57
|
}
|
60
58
|
|
@@ -1,10 +1,8 @@
|
|
1
|
-
|
2
|
-
require 'uri'
|
1
|
+
require "uri"
|
3
2
|
|
4
3
|
module Backup
|
5
4
|
module Notifier
|
6
5
|
class Prowl < Base
|
7
|
-
|
8
6
|
##
|
9
7
|
# Application name
|
10
8
|
# Tell something like your server name. Example: "Server1 Backup"
|
@@ -16,10 +14,13 @@ module Backup
|
|
16
14
|
attr_accessor :api_key
|
17
15
|
|
18
16
|
def initialize(model, &block)
|
19
|
-
@message =
|
20
|
-
|
21
|
-
|
17
|
+
@message =
|
18
|
+
lambda do |m, _|
|
19
|
+
"#{m.label} (#{m.trigger})"
|
20
|
+
end
|
21
|
+
|
22
22
|
super
|
23
|
+
|
23
24
|
instance_eval(&block) if block_given?
|
24
25
|
end
|
25
26
|
|
@@ -47,22 +48,21 @@ module Backup
|
|
47
48
|
end
|
48
49
|
|
49
50
|
def send_message(status)
|
50
|
-
uri =
|
51
|
+
uri = "https://api.prowlapp.com/publicapi/add"
|
51
52
|
status_data = status_data_for(status)
|
52
53
|
data = {
|
53
|
-
:
|
54
|
-
:
|
55
|
-
:
|
56
|
-
:
|
54
|
+
application: application,
|
55
|
+
apikey: api_key,
|
56
|
+
event: status_data[:message],
|
57
|
+
description: message.call(model, status: status_data)
|
57
58
|
}
|
58
59
|
options = {
|
59
|
-
:
|
60
|
-
:
|
60
|
+
headers: { "Content-Type" => "application/x-www-form-urlencoded" },
|
61
|
+
body: URI.encode_www_form(data)
|
61
62
|
}
|
62
|
-
options
|
63
|
+
options[:expects] = 200 # raise error if unsuccessful
|
63
64
|
Excon.post(uri, options)
|
64
65
|
end
|
65
|
-
|
66
66
|
end
|
67
67
|
end
|
68
68
|
end
|
@@ -1,10 +1,8 @@
|
|
1
|
-
|
2
|
-
require 'uri'
|
1
|
+
require "uri"
|
3
2
|
|
4
3
|
module Backup
|
5
4
|
module Notifier
|
6
5
|
class Pushover < Base
|
7
|
-
|
8
6
|
##
|
9
7
|
# The API User Token
|
10
8
|
attr_accessor :user
|
@@ -51,24 +49,23 @@ module Backup
|
|
51
49
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
52
50
|
#
|
53
51
|
def notify!(status)
|
54
|
-
send_message(message.call(model, :
|
52
|
+
send_message(message.call(model, status: status_data_for(status)))
|
55
53
|
end
|
56
54
|
|
57
55
|
def send_message(message)
|
58
|
-
uri =
|
59
|
-
data = { :
|
56
|
+
uri = "https://api.pushover.net/1/messages.json"
|
57
|
+
data = { user: user, token: token, message: message }
|
60
58
|
[:device, :title, :priority].each do |param|
|
61
59
|
val = send(param)
|
62
60
|
data.merge!(param => val) if val
|
63
61
|
end
|
64
62
|
options = {
|
65
|
-
:
|
66
|
-
:
|
63
|
+
headers: { "Content-Type" => "application/x-www-form-urlencoded" },
|
64
|
+
body: URI.encode_www_form(data)
|
67
65
|
}
|
68
|
-
options
|
66
|
+
options[:expects] = 200 # raise error if unsuccessful
|
69
67
|
Excon.post(uri, options)
|
70
68
|
end
|
71
|
-
|
72
69
|
end
|
73
70
|
end
|
74
71
|
end
|
data/lib/backup/notifier/ses.rb
CHANGED
@@ -1,13 +1,12 @@
|
|
1
|
-
|
2
|
-
require
|
1
|
+
require "aws-sdk"
|
2
|
+
require "mail"
|
3
3
|
|
4
4
|
module Backup
|
5
5
|
module Notifier
|
6
6
|
class Ses < Base
|
7
|
-
|
8
7
|
##
|
9
8
|
# Amazon Simple Email Service (SES) Credentials
|
10
|
-
attr_accessor :access_key_id, :secret_access_key
|
9
|
+
attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
|
11
10
|
|
12
11
|
##
|
13
12
|
# SES Region
|
@@ -37,7 +36,7 @@ module Backup
|
|
37
36
|
super
|
38
37
|
instance_eval(&block) if block_given?
|
39
38
|
|
40
|
-
@region ||=
|
39
|
+
@region ||= "eu-west-1"
|
41
40
|
@send_log_on ||= [:warning, :failure]
|
42
41
|
end
|
43
42
|
|
@@ -51,10 +50,15 @@ module Backup
|
|
51
50
|
private
|
52
51
|
|
53
52
|
def client
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
53
|
+
credentials = if use_iam_profile
|
54
|
+
Aws::InstanceProfileCredentials.new
|
55
|
+
else
|
56
|
+
Aws::Credentials.new(access_key_id, secret_access_key)
|
57
|
+
end
|
58
|
+
|
59
|
+
Aws::SES::Client.new(
|
60
|
+
region: region,
|
61
|
+
credentials: credentials
|
58
62
|
)
|
59
63
|
end
|
60
64
|
|
@@ -84,21 +88,35 @@ module Backup
|
|
84
88
|
email.cc = cc
|
85
89
|
email.bcc = bcc
|
86
90
|
email.reply_to = reply_to
|
87
|
-
email.subject = message.call(model, :
|
91
|
+
email.subject = message.call(model, status: status_data_for(status))
|
92
|
+
|
93
|
+
# By default, the `mail` gem doesn't include BCC in raw output, which is
|
94
|
+
# needed for SES to send to those addresses.
|
95
|
+
email[:bcc].include_in_headers = true
|
88
96
|
|
89
97
|
send_log = send_log_on.include?(status)
|
90
|
-
template = Backup::Template.new(
|
91
|
-
email.body = template.result(
|
98
|
+
template = Backup::Template.new(model: model, send_log: send_log)
|
99
|
+
email.body = template.result(sprintf("notifier/mail/%s.erb", status.to_s))
|
92
100
|
|
93
101
|
if send_log
|
94
102
|
email.convert_to_multipart
|
95
|
-
email.attachments["#{
|
96
|
-
:
|
97
|
-
:
|
103
|
+
email.attachments["#{model.time}.#{model.trigger}.log"] = {
|
104
|
+
mime_type: "text/plain;",
|
105
|
+
content: Logger.messages.map(&:formatted_lines).flatten.join("\n")
|
106
|
+
}
|
107
|
+
end
|
108
|
+
|
109
|
+
send_opts = {
|
110
|
+
raw_message: {
|
111
|
+
data: email.to_s
|
98
112
|
}
|
113
|
+
}
|
114
|
+
|
115
|
+
if email.respond_to?(:destinations)
|
116
|
+
send_opts[:destinations] = email.destinations
|
99
117
|
end
|
100
118
|
|
101
|
-
client.send_raw_email(
|
119
|
+
client.send_raw_email(send_opts)
|
102
120
|
end
|
103
121
|
end
|
104
122
|
end
|
@@ -1,11 +1,9 @@
|
|
1
|
-
|
2
|
-
require
|
3
|
-
require 'json'
|
1
|
+
require "uri"
|
2
|
+
require "json"
|
4
3
|
|
5
4
|
module Backup
|
6
5
|
module Notifier
|
7
6
|
class Slack < Base
|
8
|
-
|
9
7
|
##
|
10
8
|
# The incoming webhook url
|
11
9
|
attr_accessor :webhook_url
|
@@ -38,7 +36,7 @@ module Backup
|
|
38
36
|
instance_eval(&block) if block_given?
|
39
37
|
|
40
38
|
@send_log_on ||= [:warning, :failure]
|
41
|
-
@icon_emoji ||=
|
39
|
+
@icon_emoji ||= ":floppy_disk:"
|
42
40
|
end
|
43
41
|
|
44
42
|
private
|
@@ -62,8 +60,8 @@ module Backup
|
|
62
60
|
#
|
63
61
|
def notify!(status)
|
64
62
|
data = {
|
65
|
-
:
|
66
|
-
:
|
63
|
+
text: message.call(model, status: status_data_for(status)),
|
64
|
+
attachments: [attachment(status)]
|
67
65
|
}
|
68
66
|
[:channel, :username, :icon_emoji].each do |param|
|
69
67
|
val = send(param)
|
@@ -71,43 +69,43 @@ module Backup
|
|
71
69
|
end
|
72
70
|
|
73
71
|
options = {
|
74
|
-
:
|
75
|
-
:
|
72
|
+
headers: { "Content-Type" => "application/x-www-form-urlencoded" },
|
73
|
+
body: URI.encode_www_form(payload: JSON.dump(data))
|
76
74
|
}
|
77
|
-
options
|
75
|
+
options[:expects] = 200 # raise error if unsuccessful
|
78
76
|
Excon.post(uri, options)
|
79
77
|
end
|
80
78
|
|
81
79
|
def attachment(status)
|
82
80
|
{
|
83
|
-
:
|
84
|
-
:
|
85
|
-
:
|
86
|
-
:
|
81
|
+
fallback: "#{title(status)} - Job: #{model.label} (#{model.trigger})",
|
82
|
+
text: title(status),
|
83
|
+
color: color(status),
|
84
|
+
fields: [
|
87
85
|
{
|
88
|
-
:
|
89
|
-
:
|
90
|
-
:
|
86
|
+
title: "Job",
|
87
|
+
value: "#{model.label} (#{model.trigger})",
|
88
|
+
short: false
|
91
89
|
},
|
92
90
|
{
|
93
|
-
:
|
94
|
-
:
|
95
|
-
:
|
91
|
+
title: "Started",
|
92
|
+
value: model.started_at,
|
93
|
+
short: true
|
96
94
|
},
|
97
95
|
{
|
98
|
-
:
|
99
|
-
:
|
100
|
-
:
|
96
|
+
title: "Finished",
|
97
|
+
value: model.finished_at,
|
98
|
+
short: true
|
101
99
|
},
|
102
100
|
{
|
103
|
-
:
|
104
|
-
:
|
105
|
-
:
|
101
|
+
title: "Duration",
|
102
|
+
value: model.duration,
|
103
|
+
short: true
|
106
104
|
},
|
107
105
|
{
|
108
|
-
:
|
109
|
-
:
|
110
|
-
:
|
106
|
+
title: "Version",
|
107
|
+
value: "Backup v#{Backup::VERSION}\nRuby: #{RUBY_DESCRIPTION}",
|
108
|
+
short: false
|
111
109
|
},
|
112
110
|
log_field(status)
|
113
111
|
].compact
|
@@ -116,27 +114,28 @@ module Backup
|
|
116
114
|
|
117
115
|
def log_field(status)
|
118
116
|
send_log = send_log_on.include?(status)
|
117
|
+
return unless send_log
|
119
118
|
|
120
|
-
|
121
|
-
:
|
122
|
-
:
|
123
|
-
:
|
124
|
-
}
|
119
|
+
{
|
120
|
+
title: "Detailed Backup Log",
|
121
|
+
value: Logger.messages.map(&:formatted_lines).flatten.join("\n"),
|
122
|
+
short: false
|
123
|
+
}
|
125
124
|
end
|
126
125
|
|
127
126
|
def color(status)
|
128
127
|
case status
|
129
|
-
when :success then
|
130
|
-
when :failure then
|
131
|
-
when :warning then
|
128
|
+
when :success then "good"
|
129
|
+
when :failure then "danger"
|
130
|
+
when :warning then "warning"
|
132
131
|
end
|
133
132
|
end
|
134
133
|
|
135
134
|
def title(status)
|
136
135
|
case status
|
137
|
-
when :success then
|
138
|
-
when :failure then
|
139
|
-
when :warning then
|
136
|
+
when :success then "Backup Completed Successfully!"
|
137
|
+
when :failure then "Backup Failed!"
|
138
|
+
when :warning then "Backup Completed Successfully (with Warnings)!"
|
140
139
|
end
|
141
140
|
end
|
142
141
|
|
@@ -1,10 +1,8 @@
|
|
1
|
-
|
2
|
-
require 'twitter'
|
1
|
+
require "twitter"
|
3
2
|
|
4
3
|
module Backup
|
5
4
|
module Notifier
|
6
5
|
class Twitter < Base
|
7
|
-
|
8
6
|
##
|
9
7
|
# Twitter consumer key credentials
|
10
8
|
attr_accessor :consumer_key, :consumer_secret
|
@@ -38,7 +36,7 @@ module Backup
|
|
38
36
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
39
37
|
#
|
40
38
|
def notify!(status)
|
41
|
-
send_message(message.call(model, :
|
39
|
+
send_message(message.call(model, status: status_data_for(status)))
|
42
40
|
end
|
43
41
|
|
44
42
|
# Twitter::Client will raise an error if unsuccessful.
|
@@ -52,7 +50,6 @@ module Backup
|
|
52
50
|
|
53
51
|
client.update(message)
|
54
52
|
end
|
55
|
-
|
56
53
|
end
|
57
54
|
end
|
58
55
|
end
|
@@ -1,9 +1,6 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Notifier
|
5
3
|
class Zabbix < Base
|
6
|
-
|
7
4
|
attr_accessor :zabbix_host
|
8
5
|
|
9
6
|
attr_accessor :zabbix_port
|
@@ -19,10 +16,10 @@ module Backup
|
|
19
16
|
instance_eval(&block) if block_given?
|
20
17
|
|
21
18
|
@zabbix_host ||= Config.hostname
|
22
|
-
@zabbix_port ||=
|
23
|
-
@service_name ||= "Backup #{
|
19
|
+
@zabbix_port ||= 10_051
|
20
|
+
@service_name ||= "Backup #{model.trigger}"
|
24
21
|
@service_host ||= Config.hostname
|
25
|
-
@item_key ||=
|
22
|
+
@item_key ||= "backup_status"
|
26
23
|
end
|
27
24
|
|
28
25
|
private
|
@@ -45,18 +42,18 @@ module Backup
|
|
45
42
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
46
43
|
#
|
47
44
|
def notify!(status)
|
48
|
-
send_message(message.call(model, :
|
45
|
+
send_message(message.call(model, status: status_data_for(status)))
|
49
46
|
end
|
50
47
|
|
51
48
|
def send_message(message)
|
52
49
|
msg = [service_host, service_name, model.exit_status, message].join("\t")
|
53
|
-
cmd =
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
run("echo '#{
|
50
|
+
cmd = utility(:zabbix_sender).to_s +
|
51
|
+
" -z '#{zabbix_host}'" \
|
52
|
+
" -p '#{zabbix_port}'" \
|
53
|
+
" -s #{service_host}" \
|
54
|
+
" -k #{item_key}" \
|
55
|
+
" -o '#{msg}'"
|
56
|
+
run("echo '#{msg}' | #{cmd}")
|
60
57
|
end
|
61
58
|
end
|
62
59
|
end
|