backup 4.4.1 → 5.0.0.beta.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/LICENSE +19 -0
- data/README.md +1 -1
- data/lib/backup.rb +74 -78
- data/lib/backup/archive.rb +31 -32
- data/lib/backup/binder.rb +2 -6
- data/lib/backup/cleaner.rb +14 -18
- data/lib/backup/cli.rb +104 -108
- data/lib/backup/cloud_io/base.rb +4 -7
- data/lib/backup/cloud_io/cloud_files.rb +60 -62
- data/lib/backup/cloud_io/s3.rb +69 -76
- data/lib/backup/compressor/base.rb +4 -7
- data/lib/backup/compressor/bzip2.rb +3 -7
- data/lib/backup/compressor/custom.rb +2 -6
- data/lib/backup/compressor/gzip.rb +16 -17
- data/lib/backup/config.rb +17 -18
- data/lib/backup/config/dsl.rb +16 -17
- data/lib/backup/config/helpers.rb +10 -16
- data/lib/backup/database/base.rb +22 -21
- data/lib/backup/database/mongodb.rb +36 -37
- data/lib/backup/database/mysql.rb +40 -41
- data/lib/backup/database/openldap.rb +8 -10
- data/lib/backup/database/postgresql.rb +29 -30
- data/lib/backup/database/redis.rb +27 -30
- data/lib/backup/database/riak.rb +15 -18
- data/lib/backup/database/sqlite.rb +4 -6
- data/lib/backup/encryptor/base.rb +2 -4
- data/lib/backup/encryptor/gpg.rb +49 -59
- data/lib/backup/encryptor/open_ssl.rb +11 -14
- data/lib/backup/errors.rb +7 -12
- data/lib/backup/logger.rb +16 -18
- data/lib/backup/logger/console.rb +5 -8
- data/lib/backup/logger/fog_adapter.rb +2 -6
- data/lib/backup/logger/logfile.rb +10 -12
- data/lib/backup/logger/syslog.rb +2 -4
- data/lib/backup/model.rb +75 -40
- data/lib/backup/notifier/base.rb +24 -26
- data/lib/backup/notifier/campfire.rb +9 -11
- data/lib/backup/notifier/command.rb +0 -3
- data/lib/backup/notifier/datadog.rb +9 -12
- data/lib/backup/notifier/flowdock.rb +13 -17
- data/lib/backup/notifier/hipchat.rb +11 -13
- data/lib/backup/notifier/http_post.rb +11 -14
- data/lib/backup/notifier/mail.rb +44 -47
- data/lib/backup/notifier/nagios.rb +5 -9
- data/lib/backup/notifier/pagerduty.rb +10 -12
- data/lib/backup/notifier/prowl.rb +15 -15
- data/lib/backup/notifier/pushover.rb +7 -10
- data/lib/backup/notifier/ses.rb +34 -16
- data/lib/backup/notifier/slack.rb +39 -40
- data/lib/backup/notifier/twitter.rb +2 -5
- data/lib/backup/notifier/zabbix.rb +11 -14
- data/lib/backup/package.rb +5 -9
- data/lib/backup/packager.rb +16 -17
- data/lib/backup/pipeline.rb +17 -21
- data/lib/backup/splitter.rb +8 -11
- data/lib/backup/storage/base.rb +5 -8
- data/lib/backup/storage/cloud_files.rb +21 -23
- data/lib/backup/storage/cycler.rb +10 -15
- data/lib/backup/storage/dropbox.rb +15 -21
- data/lib/backup/storage/ftp.rb +8 -10
- data/lib/backup/storage/local.rb +5 -8
- data/lib/backup/storage/qiniu.rb +8 -8
- data/lib/backup/storage/rsync.rb +24 -26
- data/lib/backup/storage/s3.rb +27 -28
- data/lib/backup/storage/scp.rb +10 -12
- data/lib/backup/storage/sftp.rb +10 -12
- data/lib/backup/syncer/base.rb +5 -8
- data/lib/backup/syncer/cloud/base.rb +27 -30
- data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
- data/lib/backup/syncer/cloud/local_file.rb +5 -8
- data/lib/backup/syncer/cloud/s3.rb +23 -24
- data/lib/backup/syncer/rsync/base.rb +6 -10
- data/lib/backup/syncer/rsync/local.rb +1 -5
- data/lib/backup/syncer/rsync/pull.rb +6 -10
- data/lib/backup/syncer/rsync/push.rb +18 -22
- data/lib/backup/template.rb +9 -14
- data/lib/backup/utilities.rb +82 -69
- data/lib/backup/version.rb +1 -3
- metadata +100 -660
@@ -1,9 +1,6 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Notifier
|
5
3
|
class Nagios < Base
|
6
|
-
|
7
4
|
##
|
8
5
|
# Host of Nagios server to notify on backup completion.
|
9
6
|
attr_accessor :nagios_host
|
@@ -30,8 +27,8 @@ module Backup
|
|
30
27
|
|
31
28
|
@nagios_host ||= Config.hostname
|
32
29
|
@nagios_port ||= 5667
|
33
|
-
@send_nsca_cfg||= "/etc/nagios/send_nsca.cfg"
|
34
|
-
@service_name ||= "Backup #{
|
30
|
+
@send_nsca_cfg ||= "/etc/nagios/send_nsca.cfg"
|
31
|
+
@service_name ||= "Backup #{model.trigger}"
|
35
32
|
@service_host ||= Config.hostname
|
36
33
|
end
|
37
34
|
|
@@ -55,15 +52,14 @@ module Backup
|
|
55
52
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
56
53
|
#
|
57
54
|
def notify!(status)
|
58
|
-
send_message(message.call(model, :
|
55
|
+
send_message(message.call(model, status: status_data_for(status)))
|
59
56
|
end
|
60
57
|
|
61
58
|
def send_message(message)
|
62
|
-
cmd = "#{
|
59
|
+
cmd = "#{utility(:send_nsca)} -H '#{nagios_host}' -p '#{nagios_port}' -c '#{send_nsca_cfg}'"
|
63
60
|
msg = [service_host, service_name, model.exit_status, message].join("\t")
|
64
|
-
run("echo '#{
|
61
|
+
run("echo '#{msg}' | #{cmd}")
|
65
62
|
end
|
66
|
-
|
67
63
|
end
|
68
64
|
end
|
69
65
|
end
|
@@ -1,10 +1,8 @@
|
|
1
|
-
|
2
|
-
require 'pagerduty'
|
1
|
+
require "pagerduty"
|
3
2
|
|
4
3
|
module Backup
|
5
4
|
module Notifier
|
6
5
|
class PagerDuty < Base
|
7
|
-
|
8
6
|
##
|
9
7
|
# PagerDuty Service API Key. Should be a 32 character hex string.
|
10
8
|
attr_accessor :service_key
|
@@ -46,15 +44,15 @@ module Backup
|
|
46
44
|
incident_description = "Backup - #{model.label}"
|
47
45
|
incident_key = "backup/#{model.trigger}"
|
48
46
|
incident_details = {
|
49
|
-
:
|
50
|
-
:
|
51
|
-
:
|
52
|
-
:
|
53
|
-
:
|
54
|
-
:
|
55
|
-
:
|
56
|
-
:
|
57
|
-
:
|
47
|
+
incident_key: incident_key,
|
48
|
+
details: {
|
49
|
+
trigger: model.trigger,
|
50
|
+
label: model.label,
|
51
|
+
started_at: model.started_at,
|
52
|
+
finished_at: model.finished_at,
|
53
|
+
duration: model.duration,
|
54
|
+
status: status,
|
55
|
+
exception: model.exception
|
58
56
|
}
|
59
57
|
}
|
60
58
|
|
@@ -1,10 +1,8 @@
|
|
1
|
-
|
2
|
-
require 'uri'
|
1
|
+
require "uri"
|
3
2
|
|
4
3
|
module Backup
|
5
4
|
module Notifier
|
6
5
|
class Prowl < Base
|
7
|
-
|
8
6
|
##
|
9
7
|
# Application name
|
10
8
|
# Tell something like your server name. Example: "Server1 Backup"
|
@@ -16,10 +14,13 @@ module Backup
|
|
16
14
|
attr_accessor :api_key
|
17
15
|
|
18
16
|
def initialize(model, &block)
|
19
|
-
@message =
|
20
|
-
|
21
|
-
|
17
|
+
@message =
|
18
|
+
lambda do |m, _|
|
19
|
+
"#{m.label} (#{m.trigger})"
|
20
|
+
end
|
21
|
+
|
22
22
|
super
|
23
|
+
|
23
24
|
instance_eval(&block) if block_given?
|
24
25
|
end
|
25
26
|
|
@@ -47,22 +48,21 @@ module Backup
|
|
47
48
|
end
|
48
49
|
|
49
50
|
def send_message(status)
|
50
|
-
uri =
|
51
|
+
uri = "https://api.prowlapp.com/publicapi/add"
|
51
52
|
status_data = status_data_for(status)
|
52
53
|
data = {
|
53
|
-
:
|
54
|
-
:
|
55
|
-
:
|
56
|
-
:
|
54
|
+
application: application,
|
55
|
+
apikey: api_key,
|
56
|
+
event: status_data[:message],
|
57
|
+
description: message.call(model, status: status_data)
|
57
58
|
}
|
58
59
|
options = {
|
59
|
-
:
|
60
|
-
:
|
60
|
+
headers: { "Content-Type" => "application/x-www-form-urlencoded" },
|
61
|
+
body: URI.encode_www_form(data)
|
61
62
|
}
|
62
|
-
options
|
63
|
+
options[:expects] = 200 # raise error if unsuccessful
|
63
64
|
Excon.post(uri, options)
|
64
65
|
end
|
65
|
-
|
66
66
|
end
|
67
67
|
end
|
68
68
|
end
|
@@ -1,10 +1,8 @@
|
|
1
|
-
|
2
|
-
require 'uri'
|
1
|
+
require "uri"
|
3
2
|
|
4
3
|
module Backup
|
5
4
|
module Notifier
|
6
5
|
class Pushover < Base
|
7
|
-
|
8
6
|
##
|
9
7
|
# The API User Token
|
10
8
|
attr_accessor :user
|
@@ -51,24 +49,23 @@ module Backup
|
|
51
49
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
52
50
|
#
|
53
51
|
def notify!(status)
|
54
|
-
send_message(message.call(model, :
|
52
|
+
send_message(message.call(model, status: status_data_for(status)))
|
55
53
|
end
|
56
54
|
|
57
55
|
def send_message(message)
|
58
|
-
uri =
|
59
|
-
data = { :
|
56
|
+
uri = "https://api.pushover.net/1/messages.json"
|
57
|
+
data = { user: user, token: token, message: message }
|
60
58
|
[:device, :title, :priority].each do |param|
|
61
59
|
val = send(param)
|
62
60
|
data.merge!(param => val) if val
|
63
61
|
end
|
64
62
|
options = {
|
65
|
-
:
|
66
|
-
:
|
63
|
+
headers: { "Content-Type" => "application/x-www-form-urlencoded" },
|
64
|
+
body: URI.encode_www_form(data)
|
67
65
|
}
|
68
|
-
options
|
66
|
+
options[:expects] = 200 # raise error if unsuccessful
|
69
67
|
Excon.post(uri, options)
|
70
68
|
end
|
71
|
-
|
72
69
|
end
|
73
70
|
end
|
74
71
|
end
|
data/lib/backup/notifier/ses.rb
CHANGED
@@ -1,13 +1,12 @@
|
|
1
|
-
|
2
|
-
require
|
1
|
+
require "aws-sdk"
|
2
|
+
require "mail"
|
3
3
|
|
4
4
|
module Backup
|
5
5
|
module Notifier
|
6
6
|
class Ses < Base
|
7
|
-
|
8
7
|
##
|
9
8
|
# Amazon Simple Email Service (SES) Credentials
|
10
|
-
attr_accessor :access_key_id, :secret_access_key
|
9
|
+
attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
|
11
10
|
|
12
11
|
##
|
13
12
|
# SES Region
|
@@ -37,7 +36,7 @@ module Backup
|
|
37
36
|
super
|
38
37
|
instance_eval(&block) if block_given?
|
39
38
|
|
40
|
-
@region ||=
|
39
|
+
@region ||= "eu-west-1"
|
41
40
|
@send_log_on ||= [:warning, :failure]
|
42
41
|
end
|
43
42
|
|
@@ -51,10 +50,15 @@ module Backup
|
|
51
50
|
private
|
52
51
|
|
53
52
|
def client
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
53
|
+
credentials = if use_iam_profile
|
54
|
+
Aws::InstanceProfileCredentials.new
|
55
|
+
else
|
56
|
+
Aws::Credentials.new(access_key_id, secret_access_key)
|
57
|
+
end
|
58
|
+
|
59
|
+
Aws::SES::Client.new(
|
60
|
+
region: region,
|
61
|
+
credentials: credentials
|
58
62
|
)
|
59
63
|
end
|
60
64
|
|
@@ -84,21 +88,35 @@ module Backup
|
|
84
88
|
email.cc = cc
|
85
89
|
email.bcc = bcc
|
86
90
|
email.reply_to = reply_to
|
87
|
-
email.subject = message.call(model, :
|
91
|
+
email.subject = message.call(model, status: status_data_for(status))
|
92
|
+
|
93
|
+
# By default, the `mail` gem doesn't include BCC in raw output, which is
|
94
|
+
# needed for SES to send to those addresses.
|
95
|
+
email[:bcc].include_in_headers = true
|
88
96
|
|
89
97
|
send_log = send_log_on.include?(status)
|
90
|
-
template = Backup::Template.new(
|
91
|
-
email.body = template.result(
|
98
|
+
template = Backup::Template.new(model: model, send_log: send_log)
|
99
|
+
email.body = template.result(sprintf("notifier/mail/%s.erb", status.to_s))
|
92
100
|
|
93
101
|
if send_log
|
94
102
|
email.convert_to_multipart
|
95
|
-
email.attachments["#{
|
96
|
-
:
|
97
|
-
:
|
103
|
+
email.attachments["#{model.time}.#{model.trigger}.log"] = {
|
104
|
+
mime_type: "text/plain;",
|
105
|
+
content: Logger.messages.map(&:formatted_lines).flatten.join("\n")
|
106
|
+
}
|
107
|
+
end
|
108
|
+
|
109
|
+
send_opts = {
|
110
|
+
raw_message: {
|
111
|
+
data: email.to_s
|
98
112
|
}
|
113
|
+
}
|
114
|
+
|
115
|
+
if email.respond_to?(:destinations)
|
116
|
+
send_opts[:destinations] = email.destinations
|
99
117
|
end
|
100
118
|
|
101
|
-
client.send_raw_email(
|
119
|
+
client.send_raw_email(send_opts)
|
102
120
|
end
|
103
121
|
end
|
104
122
|
end
|
@@ -1,11 +1,9 @@
|
|
1
|
-
|
2
|
-
require
|
3
|
-
require 'json'
|
1
|
+
require "uri"
|
2
|
+
require "json"
|
4
3
|
|
5
4
|
module Backup
|
6
5
|
module Notifier
|
7
6
|
class Slack < Base
|
8
|
-
|
9
7
|
##
|
10
8
|
# The incoming webhook url
|
11
9
|
attr_accessor :webhook_url
|
@@ -38,7 +36,7 @@ module Backup
|
|
38
36
|
instance_eval(&block) if block_given?
|
39
37
|
|
40
38
|
@send_log_on ||= [:warning, :failure]
|
41
|
-
@icon_emoji ||=
|
39
|
+
@icon_emoji ||= ":floppy_disk:"
|
42
40
|
end
|
43
41
|
|
44
42
|
private
|
@@ -62,8 +60,8 @@ module Backup
|
|
62
60
|
#
|
63
61
|
def notify!(status)
|
64
62
|
data = {
|
65
|
-
:
|
66
|
-
:
|
63
|
+
text: message.call(model, status: status_data_for(status)),
|
64
|
+
attachments: [attachment(status)]
|
67
65
|
}
|
68
66
|
[:channel, :username, :icon_emoji].each do |param|
|
69
67
|
val = send(param)
|
@@ -71,43 +69,43 @@ module Backup
|
|
71
69
|
end
|
72
70
|
|
73
71
|
options = {
|
74
|
-
:
|
75
|
-
:
|
72
|
+
headers: { "Content-Type" => "application/x-www-form-urlencoded" },
|
73
|
+
body: URI.encode_www_form(payload: JSON.dump(data))
|
76
74
|
}
|
77
|
-
options
|
75
|
+
options[:expects] = 200 # raise error if unsuccessful
|
78
76
|
Excon.post(uri, options)
|
79
77
|
end
|
80
78
|
|
81
79
|
def attachment(status)
|
82
80
|
{
|
83
|
-
:
|
84
|
-
:
|
85
|
-
:
|
86
|
-
:
|
81
|
+
fallback: "#{title(status)} - Job: #{model.label} (#{model.trigger})",
|
82
|
+
text: title(status),
|
83
|
+
color: color(status),
|
84
|
+
fields: [
|
87
85
|
{
|
88
|
-
:
|
89
|
-
:
|
90
|
-
:
|
86
|
+
title: "Job",
|
87
|
+
value: "#{model.label} (#{model.trigger})",
|
88
|
+
short: false
|
91
89
|
},
|
92
90
|
{
|
93
|
-
:
|
94
|
-
:
|
95
|
-
:
|
91
|
+
title: "Started",
|
92
|
+
value: model.started_at,
|
93
|
+
short: true
|
96
94
|
},
|
97
95
|
{
|
98
|
-
:
|
99
|
-
:
|
100
|
-
:
|
96
|
+
title: "Finished",
|
97
|
+
value: model.finished_at,
|
98
|
+
short: true
|
101
99
|
},
|
102
100
|
{
|
103
|
-
:
|
104
|
-
:
|
105
|
-
:
|
101
|
+
title: "Duration",
|
102
|
+
value: model.duration,
|
103
|
+
short: true
|
106
104
|
},
|
107
105
|
{
|
108
|
-
:
|
109
|
-
:
|
110
|
-
:
|
106
|
+
title: "Version",
|
107
|
+
value: "Backup v#{Backup::VERSION}\nRuby: #{RUBY_DESCRIPTION}",
|
108
|
+
short: false
|
111
109
|
},
|
112
110
|
log_field(status)
|
113
111
|
].compact
|
@@ -116,27 +114,28 @@ module Backup
|
|
116
114
|
|
117
115
|
def log_field(status)
|
118
116
|
send_log = send_log_on.include?(status)
|
117
|
+
return unless send_log
|
119
118
|
|
120
|
-
|
121
|
-
:
|
122
|
-
:
|
123
|
-
:
|
124
|
-
}
|
119
|
+
{
|
120
|
+
title: "Detailed Backup Log",
|
121
|
+
value: Logger.messages.map(&:formatted_lines).flatten.join("\n"),
|
122
|
+
short: false
|
123
|
+
}
|
125
124
|
end
|
126
125
|
|
127
126
|
def color(status)
|
128
127
|
case status
|
129
|
-
when :success then
|
130
|
-
when :failure then
|
131
|
-
when :warning then
|
128
|
+
when :success then "good"
|
129
|
+
when :failure then "danger"
|
130
|
+
when :warning then "warning"
|
132
131
|
end
|
133
132
|
end
|
134
133
|
|
135
134
|
def title(status)
|
136
135
|
case status
|
137
|
-
when :success then
|
138
|
-
when :failure then
|
139
|
-
when :warning then
|
136
|
+
when :success then "Backup Completed Successfully!"
|
137
|
+
when :failure then "Backup Failed!"
|
138
|
+
when :warning then "Backup Completed Successfully (with Warnings)!"
|
140
139
|
end
|
141
140
|
end
|
142
141
|
|
@@ -1,10 +1,8 @@
|
|
1
|
-
|
2
|
-
require 'twitter'
|
1
|
+
require "twitter"
|
3
2
|
|
4
3
|
module Backup
|
5
4
|
module Notifier
|
6
5
|
class Twitter < Base
|
7
|
-
|
8
6
|
##
|
9
7
|
# Twitter consumer key credentials
|
10
8
|
attr_accessor :consumer_key, :consumer_secret
|
@@ -38,7 +36,7 @@ module Backup
|
|
38
36
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
39
37
|
#
|
40
38
|
def notify!(status)
|
41
|
-
send_message(message.call(model, :
|
39
|
+
send_message(message.call(model, status: status_data_for(status)))
|
42
40
|
end
|
43
41
|
|
44
42
|
# Twitter::Client will raise an error if unsuccessful.
|
@@ -52,7 +50,6 @@ module Backup
|
|
52
50
|
|
53
51
|
client.update(message)
|
54
52
|
end
|
55
|
-
|
56
53
|
end
|
57
54
|
end
|
58
55
|
end
|
@@ -1,9 +1,6 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Notifier
|
5
3
|
class Zabbix < Base
|
6
|
-
|
7
4
|
attr_accessor :zabbix_host
|
8
5
|
|
9
6
|
attr_accessor :zabbix_port
|
@@ -19,10 +16,10 @@ module Backup
|
|
19
16
|
instance_eval(&block) if block_given?
|
20
17
|
|
21
18
|
@zabbix_host ||= Config.hostname
|
22
|
-
@zabbix_port ||=
|
23
|
-
@service_name ||= "Backup #{
|
19
|
+
@zabbix_port ||= 10_051
|
20
|
+
@service_name ||= "Backup #{model.trigger}"
|
24
21
|
@service_host ||= Config.hostname
|
25
|
-
@item_key ||=
|
22
|
+
@item_key ||= "backup_status"
|
26
23
|
end
|
27
24
|
|
28
25
|
private
|
@@ -45,18 +42,18 @@ module Backup
|
|
45
42
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
46
43
|
#
|
47
44
|
def notify!(status)
|
48
|
-
send_message(message.call(model, :
|
45
|
+
send_message(message.call(model, status: status_data_for(status)))
|
49
46
|
end
|
50
47
|
|
51
48
|
def send_message(message)
|
52
49
|
msg = [service_host, service_name, model.exit_status, message].join("\t")
|
53
|
-
cmd =
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
run("echo '#{
|
50
|
+
cmd = utility(:zabbix_sender).to_s +
|
51
|
+
" -z '#{zabbix_host}'" \
|
52
|
+
" -p '#{zabbix_port}'" \
|
53
|
+
" -s #{service_host}" \
|
54
|
+
" -k #{item_key}" \
|
55
|
+
" -o '#{msg}'"
|
56
|
+
run("echo '#{msg}' | #{cmd}")
|
60
57
|
end
|
61
58
|
end
|
62
59
|
end
|