backup 4.3.0 → 5.0.0.beta.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (81) hide show
  1. checksums.yaml +5 -5
  2. data/LICENSE +19 -0
  3. data/README.md +13 -9
  4. data/bin/docker_test +24 -0
  5. data/lib/backup.rb +74 -78
  6. data/lib/backup/archive.rb +31 -32
  7. data/lib/backup/binder.rb +2 -6
  8. data/lib/backup/cleaner.rb +14 -18
  9. data/lib/backup/cli.rb +104 -108
  10. data/lib/backup/cloud_io/base.rb +4 -7
  11. data/lib/backup/cloud_io/cloud_files.rb +60 -62
  12. data/lib/backup/cloud_io/s3.rb +69 -76
  13. data/lib/backup/compressor/base.rb +4 -7
  14. data/lib/backup/compressor/bzip2.rb +3 -7
  15. data/lib/backup/compressor/custom.rb +2 -6
  16. data/lib/backup/compressor/gzip.rb +16 -17
  17. data/lib/backup/config.rb +17 -18
  18. data/lib/backup/config/dsl.rb +16 -17
  19. data/lib/backup/config/helpers.rb +10 -16
  20. data/lib/backup/database/base.rb +22 -21
  21. data/lib/backup/database/mongodb.rb +36 -37
  22. data/lib/backup/database/mysql.rb +40 -41
  23. data/lib/backup/database/openldap.rb +8 -10
  24. data/lib/backup/database/postgresql.rb +29 -30
  25. data/lib/backup/database/redis.rb +27 -30
  26. data/lib/backup/database/riak.rb +15 -18
  27. data/lib/backup/database/sqlite.rb +4 -6
  28. data/lib/backup/encryptor/base.rb +2 -4
  29. data/lib/backup/encryptor/gpg.rb +49 -59
  30. data/lib/backup/encryptor/open_ssl.rb +11 -14
  31. data/lib/backup/errors.rb +7 -12
  32. data/lib/backup/logger.rb +16 -18
  33. data/lib/backup/logger/console.rb +5 -8
  34. data/lib/backup/logger/fog_adapter.rb +2 -6
  35. data/lib/backup/logger/logfile.rb +10 -12
  36. data/lib/backup/logger/syslog.rb +2 -4
  37. data/lib/backup/model.rb +33 -40
  38. data/lib/backup/notifier/base.rb +24 -26
  39. data/lib/backup/notifier/campfire.rb +9 -11
  40. data/lib/backup/notifier/command.rb +3 -3
  41. data/lib/backup/notifier/datadog.rb +9 -12
  42. data/lib/backup/notifier/flowdock.rb +13 -17
  43. data/lib/backup/notifier/hipchat.rb +18 -14
  44. data/lib/backup/notifier/http_post.rb +11 -14
  45. data/lib/backup/notifier/mail.rb +42 -54
  46. data/lib/backup/notifier/nagios.rb +5 -9
  47. data/lib/backup/notifier/pagerduty.rb +10 -12
  48. data/lib/backup/notifier/prowl.rb +15 -15
  49. data/lib/backup/notifier/pushover.rb +7 -10
  50. data/lib/backup/notifier/ses.rb +52 -17
  51. data/lib/backup/notifier/slack.rb +39 -40
  52. data/lib/backup/notifier/twitter.rb +2 -5
  53. data/lib/backup/notifier/zabbix.rb +11 -14
  54. data/lib/backup/package.rb +5 -9
  55. data/lib/backup/packager.rb +16 -17
  56. data/lib/backup/pipeline.rb +17 -21
  57. data/lib/backup/splitter.rb +8 -11
  58. data/lib/backup/storage/base.rb +5 -8
  59. data/lib/backup/storage/cloud_files.rb +21 -23
  60. data/lib/backup/storage/cycler.rb +10 -15
  61. data/lib/backup/storage/dropbox.rb +15 -21
  62. data/lib/backup/storage/ftp.rb +14 -10
  63. data/lib/backup/storage/local.rb +5 -8
  64. data/lib/backup/storage/qiniu.rb +8 -8
  65. data/lib/backup/storage/rsync.rb +24 -26
  66. data/lib/backup/storage/s3.rb +27 -28
  67. data/lib/backup/storage/scp.rb +10 -12
  68. data/lib/backup/storage/sftp.rb +10 -12
  69. data/lib/backup/syncer/base.rb +5 -8
  70. data/lib/backup/syncer/cloud/base.rb +27 -30
  71. data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
  72. data/lib/backup/syncer/cloud/local_file.rb +5 -8
  73. data/lib/backup/syncer/cloud/s3.rb +23 -24
  74. data/lib/backup/syncer/rsync/base.rb +6 -10
  75. data/lib/backup/syncer/rsync/local.rb +1 -5
  76. data/lib/backup/syncer/rsync/pull.rb +6 -10
  77. data/lib/backup/syncer/rsync/push.rb +18 -22
  78. data/lib/backup/template.rb +9 -14
  79. data/lib/backup/utilities.rb +78 -69
  80. data/lib/backup/version.rb +1 -3
  81. metadata +107 -677
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Notifier
5
3
  class Error < Backup::Error; end
@@ -56,8 +54,8 @@ module Backup
56
54
  @on_failure = true if on_failure.nil?
57
55
  @max_retries ||= 10
58
56
  @retry_waitsec ||= 30
59
- @message ||= lambda do |model, data|
60
- "[#{ data[:status][:message] }] #{ model.label } (#{ model.trigger })"
57
+ @message ||= lambda do |m, data|
58
+ "[#{data[:status][:message]}] #{m.label} (#{m.trigger})"
61
59
  end
62
60
  end
63
61
 
@@ -65,22 +63,22 @@ module Backup
65
63
  # not raise any exceptions. However, each Notifier's #notify! method
66
64
  # should raise an exception if the request fails so it may be retried.
67
65
  def perform!
68
- status = case model.exit_status
69
- when 0
70
- :success if notify_on_success?
71
- when 1
72
- :warning if notify_on_success? || notify_on_warning?
73
- else
74
- :failure if notify_on_failure?
75
- end
66
+ status =
67
+ case model.exit_status
68
+ when 0
69
+ :success if notify_on_success?
70
+ when 1
71
+ :warning if notify_on_success? || notify_on_warning?
72
+ else
73
+ :failure if notify_on_failure?
74
+ end
76
75
 
77
76
  if status
78
- Logger.info "Sending notification using #{ notifier_name }..."
77
+ Logger.info "Sending notification using #{notifier_name}..."
79
78
  with_retries { notify!(status) }
80
79
  end
81
-
82
80
  rescue Exception => err
83
- Logger.error Error.wrap(err, "#{ notifier_name } Failed!")
81
+ Logger.error Error.wrap(err, "#{notifier_name} Failed!")
84
82
  end
85
83
 
86
84
  private
@@ -93,7 +91,7 @@ module Backup
93
91
  retries += 1
94
92
  raise if retries > max_retries
95
93
 
96
- Logger.info Error.wrap(err, "Retry ##{ retries } of #{ max_retries }.")
94
+ Logger.info Error.wrap(err, "Retry ##{retries} of #{max_retries}.")
97
95
  sleep(retry_waitsec)
98
96
  retry
99
97
  end
@@ -102,24 +100,24 @@ module Backup
102
100
  ##
103
101
  # Return the notifier name, with Backup namespace removed
104
102
  def notifier_name
105
- self.class.to_s.sub('Backup::', '')
103
+ self.class.to_s.sub("Backup::", "")
106
104
  end
107
105
 
108
106
  ##
109
107
  # Return status data for message creation
110
108
  def status_data_for(status)
111
109
  {
112
- :success => {
113
- :message => 'Backup::Success',
114
- :key => :success
110
+ success: {
111
+ message: "Backup::Success",
112
+ key: :success
115
113
  },
116
- :warning => {
117
- :message => 'Backup::Warning',
118
- :key => :warning
114
+ warning: {
115
+ message: "Backup::Warning",
116
+ key: :warning
119
117
  },
120
- :failure => {
121
- :message => 'Backup::Failure',
122
- :key => :failure
118
+ failure: {
119
+ message: "Backup::Failure",
120
+ key: :failure
123
121
  }
124
122
  }[status]
125
123
  end
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'json'
1
+ require "json"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Campfire < Base
7
-
8
6
  ##
9
7
  # Campfire api authentication token
10
8
  attr_accessor :api_token
@@ -42,22 +40,22 @@ module Backup
42
40
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
43
41
  #
44
42
  def notify!(status)
45
- send_message(message.call(model, :status => status_data_for(status)))
43
+ send_message(message.call(model, status: status_data_for(status)))
46
44
  end
47
45
 
48
46
  def send_message(message)
49
- uri = "https://#{ subdomain }.campfirenow.com/room/#{ room_id }/speak.json"
47
+ uri = "https://#{subdomain}.campfirenow.com/room/#{room_id}/speak.json"
50
48
  options = {
51
- :headers => { 'Content-Type' => 'application/json' },
52
- :body => JSON.dump(
53
- { :message => { :body => message, :type => 'Textmessage' } }
49
+ headers: { "Content-Type" => "application/json" },
50
+ body: JSON.dump(
51
+ message: { body: message, type: "Textmessage" }
54
52
  )
55
53
  }
56
- options.merge!(:user => api_token, :password => 'x') # Basic Auth
57
- options.merge!(:expects => 201) # raise error if unsuccessful
54
+ options[:user] = api_token
55
+ options[:password] = "x" # Basic Auth
56
+ options[:expects] = 201 # raise error if unsuccessful
58
57
  Excon.post(uri, options)
59
58
  end
60
-
61
59
  end
62
60
  end
63
61
  end
@@ -1,9 +1,6 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Notifier
5
3
  class Command < Base
6
-
7
4
  ##
8
5
  # Command to execute.
9
6
  #
@@ -23,6 +20,7 @@ module Backup
23
20
  #
24
21
  # %l - Model label
25
22
  # %t - Model trigger
23
+ # %d - Backup duration (HH:MM:SS)
26
24
  # %s - Status (success/failure/warning)
27
25
  # %v - Status verb (succeeded/failed/succeeded with warnings)
28
26
  #
@@ -73,6 +71,8 @@ module Backup
73
71
  model.label
74
72
  when "t"
75
73
  model.trigger.to_s
74
+ when "d"
75
+ model.duration
76
76
  when "v"
77
77
  status_verb(status)
78
78
  when "s"
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'dogapi'
1
+ require "dogapi"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class DataDog < Base
7
-
8
6
  ##
9
7
  # The DataDog API key
10
8
  attr_accessor :api_key
@@ -14,9 +12,9 @@ module Backup
14
12
  attr_accessor :title
15
13
 
16
14
  attr_deprecate :text,
17
- :version => '4.2',
18
- :message => 'Please use the `message` attribute. For more information '\
19
- 'see https://github.com/backup/backup/pull/698'
15
+ version: "4.2",
16
+ message: "Please use the `message` attribute. For more information "\
17
+ "see https://github.com/backup/backup/pull/698"
20
18
 
21
19
  ##
22
20
  # The timestamp for the event
@@ -49,7 +47,7 @@ module Backup
49
47
  def initialize(model, &block)
50
48
  super
51
49
  instance_eval(&block) if block_given?
52
- @title ||= "Backup #{ model.label }"
50
+ @title ||= "Backup #{model.label}"
53
51
  end
54
52
 
55
53
  private
@@ -72,7 +70,7 @@ module Backup
72
70
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
73
71
  #
74
72
  def notify!(status)
75
- msg = message.call(model, :status => status_data_for(status))
73
+ msg = message.call(model, status: status_data_for(status))
76
74
 
77
75
  hash = { alert_type: default_alert_type(status) }
78
76
  hash.store(:msg_title, @title)
@@ -96,12 +94,11 @@ module Backup
96
94
  # set alert type
97
95
  def default_alert_type(status)
98
96
  case status
99
- when :success then 'success'
100
- when :warning then 'warning'
101
- when :failure then 'error'
97
+ when :success then "success"
98
+ when :warning then "warning"
99
+ when :failure then "error"
102
100
  end
103
101
  end
104
-
105
102
  end
106
103
  end
107
104
  end
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'flowdock'
1
+ require "flowdock"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class FlowDock < Base
7
-
8
6
  ##
9
7
  # The Flowdock API token
10
8
  attr_accessor :token
@@ -68,36 +66,34 @@ module Backup
68
66
  # Flowdock::Client will raise an error if unsuccessful.
69
67
  def send_message(msg)
70
68
  client = Flowdock::Flow.new(
71
- :api_token => token, :source => source,
72
- :from => {:name => from_name, :address => from_email }
69
+ api_token: token, source: source,
70
+ from: { name: from_name, address: from_email }
73
71
  )
74
72
 
75
- client.push_to_team_inbox(:subject => subject,
76
- :content => msg,
77
- :tags => tags,
78
- :link => link )
73
+ client.push_to_team_inbox(subject: subject,
74
+ content: msg,
75
+ tags: tags,
76
+ link: link)
79
77
  end
80
78
 
81
79
  # set related tags
82
80
  def default_tags(status)
83
81
  case status
84
- when :success then ['#BackupSuccess']
85
- when :warning then ['#BackupWarning']
86
- when :failure then ['#BackupFailure']
82
+ when :success then ["#BackupSuccess"]
83
+ when :warning then ["#BackupWarning"]
84
+ when :failure then ["#BackupFailure"]
87
85
  end
88
86
  end
89
87
 
90
-
91
- #set default source
88
+ # set default source
92
89
  def default_source
93
- "Backup #{ model.label }"
90
+ "Backup #{model.label}"
94
91
  end
95
92
 
96
93
  # set default subject
97
94
  def default_subject
98
- 'Backup Notification'
95
+ "Backup Notification"
99
96
  end
100
-
101
97
  end
102
98
  end
103
99
  end
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'hipchat'
1
+ require "hipchat"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Hipchat < Base
7
-
8
6
  ##
9
7
  # The Hipchat API token
10
8
  attr_accessor :token
@@ -18,6 +16,10 @@ module Backup
18
16
  # Who the notification should appear from
19
17
  attr_accessor :from
20
18
 
19
+ ##
20
+ # Custom server URL
21
+ attr_accessor :server_url
22
+
21
23
  ##
22
24
  # The rooms that should be notified
23
25
  attr_accessor :rooms_notified
@@ -47,10 +49,10 @@ module Backup
47
49
 
48
50
  @notify_users ||= false
49
51
  @rooms_notified ||= []
50
- @success_color ||= 'yellow'
51
- @warning_color ||= 'yellow'
52
- @failure_color ||= 'yellow'
53
- @api_version ||= 'v1'
52
+ @success_color ||= "yellow"
53
+ @warning_color ||= "yellow"
54
+ @failure_color ||= "yellow"
55
+ @api_version ||= "v1"
54
56
  end
55
57
 
56
58
  private
@@ -74,24 +76,26 @@ module Backup
74
76
  #
75
77
  def notify!(status)
76
78
  status_data = status_data_for(status)
77
- msg = message.call(model, :status => status_data)
79
+ msg = message.call(model, status: status_data)
78
80
  send_message(msg, status_data[:color])
79
81
  end
80
82
 
81
83
  def client_options
82
- { api_version: @api_version }
84
+ { api_version: @api_version }.tap do |h|
85
+ h[:server_url] = server_url if server_url
86
+ end
83
87
  end
84
88
 
85
89
  # Hipchat::Client will raise an error if unsuccessful.
86
90
  def send_message(msg, color)
87
91
  client = HipChat::Client.new(token, client_options)
88
92
  rooms_to_notify.each do |room|
89
- client[room].send(from, msg, :color => color, :notify => notify_users)
93
+ client[room].send(from, msg, color: color, notify: notify_users)
90
94
  end
91
95
  end
92
96
 
93
97
  def rooms_to_notify
94
- Array(rooms_notified).map {|r| r.split(',').map(&:strip) }.flatten
98
+ Array(rooms_notified).map { |r| r.split(",").map(&:strip) }.flatten
95
99
  end
96
100
 
97
101
  def status_data_for(status)
@@ -102,9 +106,9 @@ module Backup
102
106
 
103
107
  def status_color_for(status)
104
108
  {
105
- :success => success_color,
106
- :warning => warning_color,
107
- :failure => failure_color
109
+ success: success_color,
110
+ warning: warning_color,
111
+ failure: failure_color
108
112
  }[status]
109
113
  end
110
114
  end
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'uri'
1
+ require "uri"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class HttpPost < Base
7
-
8
6
  ##
9
7
  # URI to post notification to.
10
8
  #
@@ -95,23 +93,22 @@ module Backup
95
93
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
96
94
  #
97
95
  def notify!(status)
98
- msg = message.call(model, :status => status_data_for(status))
96
+ msg = message.call(model, status: status_data_for(status))
99
97
 
100
98
  opts = {
101
- :headers => { 'User-Agent' => "Backup/#{ VERSION }" }.
102
- merge(headers).reject {|k,v| v.nil? }.
103
- merge('Content-Type' => 'application/x-www-form-urlencoded'),
104
- :body => URI.encode_www_form({ 'message' => msg }.
105
- merge(params).reject {|k,v| v.nil? }.
106
- merge('status' => status.to_s)),
107
- :expects => success_codes # raise error if unsuccessful
99
+ headers: { "User-Agent" => "Backup/#{VERSION}" }
100
+ .merge(headers).reject { |_, value| value.nil? }
101
+ .merge("Content-Type" => "application/x-www-form-urlencoded"),
102
+ body: URI.encode_www_form({ "message" => msg }
103
+ .merge(params).reject { |_, value| value.nil? }
104
+ .merge("status" => status.to_s)),
105
+ expects: success_codes # raise error if unsuccessful
108
106
  }
109
- opts.merge!(:ssl_verify_peer => ssl_verify_peer) unless ssl_verify_peer.nil?
110
- opts.merge!(:ssl_ca_file => ssl_ca_file) if ssl_ca_file
107
+ opts[:ssl_verify_peer] = ssl_verify_peer unless ssl_verify_peer.nil?
108
+ opts[:ssl_ca_file] = ssl_ca_file if ssl_ca_file
111
109
 
112
110
  Excon.post(uri, opts)
113
111
  end
114
-
115
112
  end
116
113
  end
117
114
  end
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'mail'
1
+ require "mail"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Mail < Base
7
-
8
6
  ##
9
7
  # Mail delivery method to be used by the Mail gem.
10
8
  #
@@ -161,17 +159,17 @@ module Backup
161
159
  #
162
160
  def notify!(status)
163
161
  email = new_email
164
- email.subject = message.call(model, :status => status_data_for(status))
162
+ email.subject = message.call(model, status: status_data_for(status))
165
163
 
166
164
  send_log = send_log_on.include?(status)
167
- template = Backup::Template.new({ :model => model, :send_log => send_log })
168
- email.body = template.result('notifier/mail/%s.erb' % status.to_s)
165
+ template = Backup::Template.new(model: model, send_log: send_log)
166
+ email.body = template.result(sprintf("notifier/mail/%s.erb", status.to_s))
169
167
 
170
168
  if send_log
171
169
  email.convert_to_multipart
172
- email.attachments["#{ model.time }.#{ model.trigger }.log"] = {
173
- :mime_type => 'text/plain;',
174
- :content => Logger.messages.map(&:formatted_lines).flatten.join("\n")
170
+ email.attachments["#{model.time}.#{model.trigger}.log"] = {
171
+ mime_type: "text/plain;",
172
+ content: Logger.messages.map(&:formatted_lines).flatten.join("\n")
175
173
  }
176
174
  end
177
175
 
@@ -182,38 +180,43 @@ module Backup
182
180
  # Configures the Mail gem by setting the defaults.
183
181
  # Creates and returns a new email, based on the @delivery_method used.
184
182
  def new_email
185
- method = %w{ smtp sendmail exim file test }.
186
- index(@delivery_method.to_s) ? @delivery_method.to_s : 'smtp'
183
+ method = %w[smtp sendmail exim file test]
184
+ .index(@delivery_method.to_s) ? @delivery_method.to_s : "smtp"
187
185
 
188
186
  options =
189
- case method
190
- when 'smtp'
191
- { :address => @address,
192
- :port => @port,
193
- :domain => @domain,
194
- :user_name => @user_name,
195
- :password => @password,
196
- :authentication => @authentication,
197
- :enable_starttls_auto => @encryption == :starttls,
198
- :openssl_verify_mode => @openssl_verify_mode,
199
- :ssl => @encryption == :ssl,
200
- :tls => @encryption == :tls
201
- }
202
- when 'sendmail'
203
- opts = {}
204
- opts.merge!(:location => utility(:sendmail))
205
- opts.merge!(:arguments => @sendmail_args) if @sendmail_args
206
- opts
207
- when 'exim'
208
- opts = {}
209
- opts.merge!(:location => utility(:exim))
210
- opts.merge!(:arguments => @exim_args) if @exim_args
211
- opts
212
- when 'file'
213
- @mail_folder ||= File.join(Config.root_path, 'emails')
214
- { :location => File.expand_path(@mail_folder) }
215
- when 'test' then {}
216
- end
187
+ case method
188
+ when "smtp"
189
+ opts = {
190
+ address: @address,
191
+ port: @port,
192
+ user_name: @user_name,
193
+ password: @password,
194
+ authentication: @authentication,
195
+ enable_starttls_auto: @encryption == :starttls,
196
+ openssl_verify_mode: @openssl_verify_mode,
197
+ ssl: @encryption == :ssl,
198
+ tls: @encryption == :tls
199
+ }
200
+
201
+ # Don't override default domain setting if domain not applicable.
202
+ # ref https://github.com/mikel/mail/blob/2.6.3/lib/mail/network/delivery_methods/smtp.rb#L82
203
+ opts[:domain] = @domain if @domain
204
+ opts
205
+ when "sendmail"
206
+ opts = {}
207
+ opts[:location] = utility(:sendmail)
208
+ opts[:arguments] = @sendmail_args if @sendmail_args
209
+ opts
210
+ when "exim"
211
+ opts = {}
212
+ opts[:location] = utility(:exim)
213
+ opts[:arguments] = @exim_args if @exim_args
214
+ opts
215
+ when "file"
216
+ @mail_folder ||= File.join(Config.root_path, "emails")
217
+ { location: File.expand_path(@mail_folder) }
218
+ when "test" then {}
219
+ end
217
220
 
218
221
  email = ::Mail.new
219
222
  email.delivery_method method.to_sym, options
@@ -224,21 +227,6 @@ module Backup
224
227
  email.reply_to = reply_to
225
228
  email
226
229
  end
227
-
228
- end
229
- end
230
- end
231
-
232
- # Patch mail v2.5.4 Exim delivery method
233
- # https://github.com/backup/backup/issues/446
234
- # https://github.com/mikel/mail/pull/546
235
- module Mail
236
- class Exim
237
- def self.call(path, arguments, destinations, encoded_message)
238
- popen "#{path} #{arguments}" do |io|
239
- io.puts encoded_message.to_lf
240
- io.flush
241
- end
242
230
  end
243
231
  end
244
232
  end