backup-ssh 4.1.10 → 4.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +10 -6
- data/lib/backup.rb +3 -0
- data/lib/backup/config/dsl.rb +2 -2
- data/lib/backup/database/mongodb.rb +2 -1
- data/lib/backup/database/mysql.rb +2 -2
- data/lib/backup/database/postgresql.rb +1 -1
- data/lib/backup/model.rb +26 -1
- data/lib/backup/notifier/base.rb +30 -0
- data/lib/backup/notifier/campfire.rb +1 -7
- data/lib/backup/notifier/command.rb +102 -0
- data/lib/backup/notifier/datadog.rb +11 -20
- data/lib/backup/notifier/flowdock.rb +6 -5
- data/lib/backup/notifier/hipchat.rb +33 -8
- data/lib/backup/notifier/http_post.rb +2 -7
- data/lib/backup/notifier/mail.rb +26 -15
- data/lib/backup/notifier/nagios.rb +3 -8
- data/lib/backup/notifier/prowl.rb +8 -9
- data/lib/backup/notifier/pushover.rb +1 -7
- data/lib/backup/notifier/ses.rb +19 -8
- data/lib/backup/notifier/slack.rb +4 -10
- data/lib/backup/notifier/twitter.rb +1 -7
- data/lib/backup/notifier/zabbix.rb +1 -6
- data/lib/backup/package.rb +4 -0
- data/lib/backup/packager.rb +8 -2
- data/lib/backup/storage/base.rb +15 -3
- data/lib/backup/storage/cycler.rb +24 -14
- data/lib/backup/storage/dropbox.rb +0 -24
- data/lib/backup/storage/ftp.rb +15 -1
- data/lib/backup/storage/qiniu.rb +65 -0
- data/lib/backup/storage/s3.rb +3 -2
- data/lib/backup/syncer/rsync/base.rb +7 -1
- data/lib/backup/version.rb +1 -1
- data/templates/cli/databases/mongodb +1 -1
- data/templates/cli/notifiers/command +32 -0
- data/templates/cli/notifiers/hipchat +1 -0
- data/templates/cli/notifiers/mail +3 -0
- data/templates/cli/storages/dropbox +1 -0
- data/templates/cli/storages/ftp +1 -0
- data/templates/cli/storages/local +1 -0
- data/templates/cli/storages/qiniu +12 -0
- data/templates/cli/storages/s3 +2 -0
- data/templates/cli/storages/scp +1 -0
- data/templates/cli/storages/sftp +1 -0
- metadata +79 -46
- data/LICENSE.md +0 -24
- data/lib/backup/storage/ninefold.rb +0 -74
- data/templates/cli/storages/ninefold +0 -9
@@ -95,18 +95,13 @@ module Backup
|
|
95
95
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
96
96
|
#
|
97
97
|
def notify!(status)
|
98
|
-
|
99
|
-
when :success then '[Backup::Success]'
|
100
|
-
when :failure then '[Backup::Failure]'
|
101
|
-
when :warning then '[Backup::Warning]'
|
102
|
-
end
|
103
|
-
message = "#{ tag } #{ model.label } (#{ model.trigger })"
|
98
|
+
msg = message.call(model, :status => status_data_for(status))
|
104
99
|
|
105
100
|
opts = {
|
106
101
|
:headers => { 'User-Agent' => "Backup/#{ VERSION }" }.
|
107
102
|
merge(headers).reject {|k,v| v.nil? }.
|
108
103
|
merge('Content-Type' => 'application/x-www-form-urlencoded'),
|
109
|
-
:body => URI.encode_www_form({ 'message' =>
|
104
|
+
:body => URI.encode_www_form({ 'message' => msg }.
|
110
105
|
merge(params).reject {|k,v| v.nil? }.
|
111
106
|
merge('status' => status.to_s)),
|
112
107
|
:expects => success_codes # raise error if unsuccessful
|
data/lib/backup/notifier/mail.rb
CHANGED
@@ -37,6 +37,18 @@ module Backup
|
|
37
37
|
# Receiver Email Address
|
38
38
|
attr_accessor :to
|
39
39
|
|
40
|
+
##
|
41
|
+
# CC receiver Email Address
|
42
|
+
attr_accessor :cc
|
43
|
+
|
44
|
+
##
|
45
|
+
# BCC receiver Email Address
|
46
|
+
attr_accessor :bcc
|
47
|
+
|
48
|
+
##
|
49
|
+
# Set reply to email address
|
50
|
+
attr_accessor :reply_to
|
51
|
+
|
40
52
|
##
|
41
53
|
# SMTP Server Address
|
42
54
|
attr_accessor :address
|
@@ -148,14 +160,8 @@ module Backup
|
|
148
160
|
# : backup log, if `on_failure` is `true`.
|
149
161
|
#
|
150
162
|
def notify!(status)
|
151
|
-
tag = case status
|
152
|
-
when :success then '[Backup::Success]'
|
153
|
-
when :warning then '[Backup::Warning]'
|
154
|
-
when :failure then '[Backup::Failure]'
|
155
|
-
end
|
156
|
-
|
157
163
|
email = new_email
|
158
|
-
email.subject =
|
164
|
+
email.subject = message.call(model, :status => status_data_for(status))
|
159
165
|
|
160
166
|
send_log = send_log_on.include?(status)
|
161
167
|
template = Backup::Template.new({ :model => model, :send_log => send_log })
|
@@ -182,9 +188,9 @@ module Backup
|
|
182
188
|
options =
|
183
189
|
case method
|
184
190
|
when 'smtp'
|
185
|
-
|
191
|
+
opts = {
|
192
|
+
:address => @address,
|
186
193
|
:port => @port,
|
187
|
-
:domain => @domain,
|
188
194
|
:user_name => @user_name,
|
189
195
|
:password => @password,
|
190
196
|
:authentication => @authentication,
|
@@ -193,6 +199,11 @@ module Backup
|
|
193
199
|
:ssl => @encryption == :ssl,
|
194
200
|
:tls => @encryption == :tls
|
195
201
|
}
|
202
|
+
|
203
|
+
# Don't override default domain setting if domain not applicable.
|
204
|
+
# ref https://github.com/mikel/mail/blob/2.6.3/lib/mail/network/delivery_methods/smtp.rb#L82
|
205
|
+
opts[:domain] = @domain if @domain
|
206
|
+
opts
|
196
207
|
when 'sendmail'
|
197
208
|
opts = {}
|
198
209
|
opts.merge!(:location => utility(:sendmail))
|
@@ -209,13 +220,13 @@ module Backup
|
|
209
220
|
when 'test' then {}
|
210
221
|
end
|
211
222
|
|
212
|
-
::Mail.defaults do
|
213
|
-
delivery_method method.to_sym, options
|
214
|
-
end
|
215
|
-
|
216
223
|
email = ::Mail.new
|
217
|
-
email.
|
218
|
-
email.
|
224
|
+
email.delivery_method method.to_sym, options
|
225
|
+
email.to = to
|
226
|
+
email.from = from
|
227
|
+
email.cc = cc
|
228
|
+
email.bcc = bcc
|
229
|
+
email.reply_to = reply_to
|
219
230
|
email
|
220
231
|
end
|
221
232
|
|
@@ -14,8 +14,8 @@ module Backup
|
|
14
14
|
|
15
15
|
##
|
16
16
|
# Nagios nrpe configuration file.
|
17
|
-
attr_accessor :send_nsca_cfg
|
18
|
-
|
17
|
+
attr_accessor :send_nsca_cfg
|
18
|
+
|
19
19
|
##
|
20
20
|
# Name of the Nagios service for the backup check.
|
21
21
|
attr_accessor :service_name
|
@@ -55,12 +55,7 @@ module Backup
|
|
55
55
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
56
56
|
#
|
57
57
|
def notify!(status)
|
58
|
-
message
|
59
|
-
when :success then 'Completed Successfully'
|
60
|
-
when :warning then 'Completed Successfully (with Warnings)'
|
61
|
-
when :failure then 'Failed'
|
62
|
-
end
|
63
|
-
send_message("#{ message } in #{ model.duration }")
|
58
|
+
send_message(message.call(model, :status => status_data_for(status)))
|
64
59
|
end
|
65
60
|
|
66
61
|
def send_message(message)
|
@@ -16,6 +16,9 @@ module Backup
|
|
16
16
|
attr_accessor :api_key
|
17
17
|
|
18
18
|
def initialize(model, &block)
|
19
|
+
@message = lambda do |model, data|
|
20
|
+
"#{ model.label } (#{ model.trigger })"
|
21
|
+
end
|
19
22
|
super
|
20
23
|
instance_eval(&block) if block_given?
|
21
24
|
end
|
@@ -40,21 +43,17 @@ module Backup
|
|
40
43
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
41
44
|
#
|
42
45
|
def notify!(status)
|
43
|
-
|
44
|
-
when :success then '[Backup::Success]'
|
45
|
-
when :warning then '[Backup::Warning]'
|
46
|
-
when :failure then '[Backup::Failure]'
|
47
|
-
end
|
48
|
-
send_message(tag)
|
46
|
+
send_message(status)
|
49
47
|
end
|
50
48
|
|
51
|
-
def send_message(
|
49
|
+
def send_message(status)
|
52
50
|
uri = 'https://api.prowlapp.com/publicapi/add'
|
51
|
+
status_data = status_data_for(status)
|
53
52
|
data = {
|
54
53
|
:application => application,
|
55
54
|
:apikey => api_key,
|
56
|
-
:event => message,
|
57
|
-
:description =>
|
55
|
+
:event => status_data[:message],
|
56
|
+
:description => message.call(model, :status => status_data)
|
58
57
|
}
|
59
58
|
options = {
|
60
59
|
:headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
|
@@ -51,13 +51,7 @@ module Backup
|
|
51
51
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
52
52
|
#
|
53
53
|
def notify!(status)
|
54
|
-
|
55
|
-
when :success then '[Backup::Success]'
|
56
|
-
when :failure then '[Backup::Failure]'
|
57
|
-
when :warning then '[Backup::Warning]'
|
58
|
-
end
|
59
|
-
message = "#{ tag } #{ model.label } (#{ model.trigger })"
|
60
|
-
send_message(message)
|
54
|
+
send_message(message.call(model, :status => status_data_for(status)))
|
61
55
|
end
|
62
56
|
|
63
57
|
def send_message(message)
|
data/lib/backup/notifier/ses.rb
CHANGED
@@ -21,6 +21,18 @@ module Backup
|
|
21
21
|
# Receiver Email Address
|
22
22
|
attr_accessor :to
|
23
23
|
|
24
|
+
##
|
25
|
+
# CC receiver Email Address
|
26
|
+
attr_accessor :cc
|
27
|
+
|
28
|
+
##
|
29
|
+
# BCC receiver Email Address
|
30
|
+
attr_accessor :bcc
|
31
|
+
|
32
|
+
##
|
33
|
+
# Set reply to email address
|
34
|
+
attr_accessor :reply_to
|
35
|
+
|
24
36
|
def initialize(model, &block)
|
25
37
|
super
|
26
38
|
instance_eval(&block) if block_given?
|
@@ -66,14 +78,13 @@ module Backup
|
|
66
78
|
# : backup log, if `on_failure` is `true`.
|
67
79
|
#
|
68
80
|
def notify!(status)
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
email
|
76
|
-
email.subject = "#{ tag } #{ model.label } (#{ model.trigger })"
|
81
|
+
email = ::Mail.new
|
82
|
+
email.to = to
|
83
|
+
email.from = from
|
84
|
+
email.cc = cc
|
85
|
+
email.bcc = bcc
|
86
|
+
email.reply_to = reply_to
|
87
|
+
email.subject = message.call(model, :status => status_data_for(status))
|
77
88
|
|
78
89
|
send_log = send_log_on.include?(status)
|
79
90
|
template = Backup::Template.new({ :model => model, :send_log => send_log })
|
@@ -61,21 +61,15 @@ module Backup
|
|
61
61
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
62
62
|
#
|
63
63
|
def notify!(status)
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
end
|
69
|
-
message = "#{ tag } #{ model.label } (#{ model.trigger })"
|
70
|
-
|
71
|
-
data = { :text => message }
|
64
|
+
data = {
|
65
|
+
:text => message.call(model, :status => status_data_for(status)),
|
66
|
+
:attachments => [attachment(status)]
|
67
|
+
}
|
72
68
|
[:channel, :username, :icon_emoji].each do |param|
|
73
69
|
val = send(param)
|
74
70
|
data.merge!(param => val) if val
|
75
71
|
end
|
76
72
|
|
77
|
-
data.merge!(:attachments => [attachment(status)])
|
78
|
-
|
79
73
|
options = {
|
80
74
|
:headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
|
81
75
|
:body => URI.encode_www_form(:payload => JSON.dump(data))
|
@@ -38,13 +38,7 @@ module Backup
|
|
38
38
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
39
39
|
#
|
40
40
|
def notify!(status)
|
41
|
-
|
42
|
-
when :success then '[Backup::Success]'
|
43
|
-
when :warning then '[Backup::Warning]'
|
44
|
-
when :failure then '[Backup::Failure]'
|
45
|
-
end
|
46
|
-
message = "#{ tag } #{ model.label } (#{ model.trigger }) (@ #{ model.time })"
|
47
|
-
send_message(message)
|
41
|
+
send_message(message.call(model, :status => status_data_for(status)))
|
48
42
|
end
|
49
43
|
|
50
44
|
# Twitter::Client will raise an error if unsuccessful.
|
@@ -45,12 +45,7 @@ module Backup
|
|
45
45
|
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
46
46
|
#
|
47
47
|
def notify!(status)
|
48
|
-
message
|
49
|
-
when :success then 'Completed Successfully'
|
50
|
-
when :warning then 'Completed Successfully (with Warnings)'
|
51
|
-
when :failure then 'Failed'
|
52
|
-
end
|
53
|
-
send_message("#{ message } in #{ model.duration }")
|
48
|
+
send_message(message.call(model, :status => status_data_for(status)))
|
54
49
|
end
|
55
50
|
|
56
51
|
def send_message(message)
|
data/lib/backup/package.rb
CHANGED
data/lib/backup/packager.rb
CHANGED
@@ -42,8 +42,11 @@ module Backup
|
|
42
42
|
# The command's output will then be either piped to the Encryptor
|
43
43
|
# or the Splitter (if no Encryptor), or through `cat` into the final
|
44
44
|
# output file if neither are configured.
|
45
|
-
@pipeline
|
46
|
-
|
45
|
+
@pipeline.add(
|
46
|
+
"#{ utility(:tar) } -cf - " +
|
47
|
+
"-C '#{ Config.tmp_path }' '#{ @package.trigger }'",
|
48
|
+
tar_success_codes
|
49
|
+
)
|
47
50
|
|
48
51
|
##
|
49
52
|
# If an Encryptor was configured, it will be called first
|
@@ -96,6 +99,9 @@ module Backup
|
|
96
99
|
stack.shift
|
97
100
|
end
|
98
101
|
|
102
|
+
def tar_success_codes
|
103
|
+
gnu_tar? ? [0, 1] : [0]
|
104
|
+
end
|
99
105
|
end
|
100
106
|
end
|
101
107
|
end
|
data/lib/backup/storage/base.rb
CHANGED
@@ -10,8 +10,18 @@ module Backup
|
|
10
10
|
attr_accessor :path
|
11
11
|
|
12
12
|
##
|
13
|
-
#
|
14
|
-
#
|
13
|
+
# Number of backups to keep or time until which to keep.
|
14
|
+
#
|
15
|
+
# If an Integer is given it sets the limit to how many backups to keep in
|
16
|
+
# the remote location. If exceeded, the oldest will be removed to make
|
17
|
+
# room for the newest.
|
18
|
+
#
|
19
|
+
# If a Time object is given it will remove backups _older_ than the given
|
20
|
+
# date.
|
21
|
+
#
|
22
|
+
# @!attribute [rw] keep
|
23
|
+
# @param [Integer|Time]
|
24
|
+
# @return [Integer|Time]
|
15
25
|
attr_accessor :keep
|
16
26
|
|
17
27
|
attr_reader :model, :package, :storage_id
|
@@ -33,7 +43,9 @@ module Backup
|
|
33
43
|
def perform!
|
34
44
|
Logger.info "#{ storage_name } Started..."
|
35
45
|
transfer!
|
36
|
-
|
46
|
+
if respond_to?(:cycle!, true) && (keep.to_i > 0 || keep.is_a?(Time))
|
47
|
+
cycle!
|
48
|
+
end
|
37
49
|
Logger.info "#{ storage_name } Finished!"
|
38
50
|
end
|
39
51
|
|
@@ -14,24 +14,34 @@ module Backup
|
|
14
14
|
Logger.info 'Cycling Started...'
|
15
15
|
|
16
16
|
packages = yaml_load.unshift(package)
|
17
|
-
|
17
|
+
cycled_packages = []
|
18
18
|
|
19
|
-
if
|
20
|
-
packages.
|
21
|
-
|
22
|
-
remove!(pkg) unless pkg.no_cycle
|
23
|
-
rescue => err
|
24
|
-
Logger.warn Error.wrap(err, <<-EOS)
|
25
|
-
There was a problem removing the following package:
|
26
|
-
Trigger: #{pkg.trigger} :: Dated: #{pkg.time}
|
27
|
-
Package included the following #{ pkg.filenames.count } file(s):
|
28
|
-
#{ pkg.filenames.join("\n") }
|
29
|
-
EOS
|
30
|
-
end
|
19
|
+
if keep.is_a?(Date) || keep.is_a?(Time)
|
20
|
+
cycled_packages = packages.select do |p|
|
21
|
+
p.time_as_object < keep.to_time
|
31
22
|
end
|
23
|
+
else
|
24
|
+
excess = packages.count - keep.to_i
|
25
|
+
cycled_packages = packages.last(excess) if excess > 0
|
32
26
|
end
|
33
27
|
|
34
|
-
|
28
|
+
saved_packages = packages - cycled_packages
|
29
|
+
cycled_packages.each { |package| delete_package package }
|
30
|
+
|
31
|
+
yaml_save(saved_packages)
|
32
|
+
end
|
33
|
+
|
34
|
+
def delete_package(package)
|
35
|
+
begin
|
36
|
+
remove!(package) unless package.no_cycle
|
37
|
+
rescue => err
|
38
|
+
Logger.warn Error.wrap(err, <<-EOS)
|
39
|
+
There was a problem removing the following package:
|
40
|
+
Trigger: #{package.trigger} :: Dated: #{package.time}
|
41
|
+
Package included the following #{ package.filenames.count } file(s):
|
42
|
+
#{ package.filenames.join("\n") }
|
43
|
+
EOS
|
44
|
+
end
|
35
45
|
end
|
36
46
|
|
37
47
|
# Returns path to the YAML data file.
|
@@ -210,27 +210,3 @@ module Backup
|
|
210
210
|
end
|
211
211
|
end
|
212
212
|
end
|
213
|
-
|
214
|
-
# Patch for dropbox-ruby-sdk-1.5.1
|
215
|
-
class DropboxClient
|
216
|
-
class ChunkedUploader
|
217
|
-
def upload(chunk_size = 1024**2 * 4)
|
218
|
-
while @offset < @total_size
|
219
|
-
@file_obj.seek(@offset) unless @file_obj.pos == @offset
|
220
|
-
data = @file_obj.read(chunk_size)
|
221
|
-
|
222
|
-
begin
|
223
|
-
resp = @client.parse_response(
|
224
|
-
@client.partial_chunked_upload(data, @upload_id, @offset)
|
225
|
-
)
|
226
|
-
rescue DropboxError => err
|
227
|
-
resp = JSON.parse(err.http_response.body) rescue {}
|
228
|
-
raise err unless resp['offset']
|
229
|
-
end
|
230
|
-
|
231
|
-
@offset = resp['offset']
|
232
|
-
@upload_id ||= resp['upload_id']
|
233
|
-
end
|
234
|
-
end
|
235
|
-
end
|
236
|
-
end
|
data/lib/backup/storage/ftp.rb
CHANGED
@@ -15,15 +15,25 @@ module Backup
|
|
15
15
|
attr_accessor :ip, :port
|
16
16
|
|
17
17
|
##
|
18
|
-
#
|
18
|
+
# Use passive mode?
|
19
19
|
attr_accessor :passive_mode
|
20
20
|
|
21
|
+
##
|
22
|
+
# Configure connection open and read timeouts.
|
23
|
+
# Net::FTP's open_timeout and read_timeout will both be configured using
|
24
|
+
# this setting.
|
25
|
+
# @!attribute [rw] timeout
|
26
|
+
# @param [Integer|Float]
|
27
|
+
# @return [Integer|Float]
|
28
|
+
attr_accessor :timeout
|
29
|
+
|
21
30
|
def initialize(model, storage_id = nil)
|
22
31
|
super
|
23
32
|
|
24
33
|
@port ||= 21
|
25
34
|
@path ||= 'backups'
|
26
35
|
@passive_mode ||= false
|
36
|
+
@timeout ||= nil
|
27
37
|
path.sub!(/^~\//, '')
|
28
38
|
end
|
29
39
|
|
@@ -42,6 +52,10 @@ module Backup
|
|
42
52
|
end; Net::FTP.send(:const_set, :FTP_PORT, port)
|
43
53
|
|
44
54
|
Net::FTP.open(ip, username, password) do |ftp|
|
55
|
+
if timeout
|
56
|
+
ftp.open_timeout = timeout
|
57
|
+
ftp.read_timeout = timeout
|
58
|
+
end
|
45
59
|
ftp.passive = true if passive_mode
|
46
60
|
yield ftp
|
47
61
|
end
|