backup 4.3.0 → 5.0.0.beta.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (81) hide show
  1. checksums.yaml +5 -5
  2. data/LICENSE +19 -0
  3. data/README.md +13 -9
  4. data/bin/docker_test +24 -0
  5. data/lib/backup.rb +74 -78
  6. data/lib/backup/archive.rb +31 -32
  7. data/lib/backup/binder.rb +2 -6
  8. data/lib/backup/cleaner.rb +14 -18
  9. data/lib/backup/cli.rb +104 -108
  10. data/lib/backup/cloud_io/base.rb +4 -7
  11. data/lib/backup/cloud_io/cloud_files.rb +60 -62
  12. data/lib/backup/cloud_io/s3.rb +69 -76
  13. data/lib/backup/compressor/base.rb +4 -7
  14. data/lib/backup/compressor/bzip2.rb +3 -7
  15. data/lib/backup/compressor/custom.rb +2 -6
  16. data/lib/backup/compressor/gzip.rb +16 -17
  17. data/lib/backup/config.rb +17 -18
  18. data/lib/backup/config/dsl.rb +16 -17
  19. data/lib/backup/config/helpers.rb +10 -16
  20. data/lib/backup/database/base.rb +22 -21
  21. data/lib/backup/database/mongodb.rb +36 -37
  22. data/lib/backup/database/mysql.rb +40 -41
  23. data/lib/backup/database/openldap.rb +8 -10
  24. data/lib/backup/database/postgresql.rb +29 -30
  25. data/lib/backup/database/redis.rb +27 -30
  26. data/lib/backup/database/riak.rb +15 -18
  27. data/lib/backup/database/sqlite.rb +4 -6
  28. data/lib/backup/encryptor/base.rb +2 -4
  29. data/lib/backup/encryptor/gpg.rb +49 -59
  30. data/lib/backup/encryptor/open_ssl.rb +11 -14
  31. data/lib/backup/errors.rb +7 -12
  32. data/lib/backup/logger.rb +16 -18
  33. data/lib/backup/logger/console.rb +5 -8
  34. data/lib/backup/logger/fog_adapter.rb +2 -6
  35. data/lib/backup/logger/logfile.rb +10 -12
  36. data/lib/backup/logger/syslog.rb +2 -4
  37. data/lib/backup/model.rb +33 -40
  38. data/lib/backup/notifier/base.rb +24 -26
  39. data/lib/backup/notifier/campfire.rb +9 -11
  40. data/lib/backup/notifier/command.rb +3 -3
  41. data/lib/backup/notifier/datadog.rb +9 -12
  42. data/lib/backup/notifier/flowdock.rb +13 -17
  43. data/lib/backup/notifier/hipchat.rb +18 -14
  44. data/lib/backup/notifier/http_post.rb +11 -14
  45. data/lib/backup/notifier/mail.rb +42 -54
  46. data/lib/backup/notifier/nagios.rb +5 -9
  47. data/lib/backup/notifier/pagerduty.rb +10 -12
  48. data/lib/backup/notifier/prowl.rb +15 -15
  49. data/lib/backup/notifier/pushover.rb +7 -10
  50. data/lib/backup/notifier/ses.rb +52 -17
  51. data/lib/backup/notifier/slack.rb +39 -40
  52. data/lib/backup/notifier/twitter.rb +2 -5
  53. data/lib/backup/notifier/zabbix.rb +11 -14
  54. data/lib/backup/package.rb +5 -9
  55. data/lib/backup/packager.rb +16 -17
  56. data/lib/backup/pipeline.rb +17 -21
  57. data/lib/backup/splitter.rb +8 -11
  58. data/lib/backup/storage/base.rb +5 -8
  59. data/lib/backup/storage/cloud_files.rb +21 -23
  60. data/lib/backup/storage/cycler.rb +10 -15
  61. data/lib/backup/storage/dropbox.rb +15 -21
  62. data/lib/backup/storage/ftp.rb +14 -10
  63. data/lib/backup/storage/local.rb +5 -8
  64. data/lib/backup/storage/qiniu.rb +8 -8
  65. data/lib/backup/storage/rsync.rb +24 -26
  66. data/lib/backup/storage/s3.rb +27 -28
  67. data/lib/backup/storage/scp.rb +10 -12
  68. data/lib/backup/storage/sftp.rb +10 -12
  69. data/lib/backup/syncer/base.rb +5 -8
  70. data/lib/backup/syncer/cloud/base.rb +27 -30
  71. data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
  72. data/lib/backup/syncer/cloud/local_file.rb +5 -8
  73. data/lib/backup/syncer/cloud/s3.rb +23 -24
  74. data/lib/backup/syncer/rsync/base.rb +6 -10
  75. data/lib/backup/syncer/rsync/local.rb +1 -5
  76. data/lib/backup/syncer/rsync/pull.rb +6 -10
  77. data/lib/backup/syncer/rsync/push.rb +18 -22
  78. data/lib/backup/template.rb +9 -14
  79. data/lib/backup/utilities.rb +78 -69
  80. data/lib/backup/version.rb +1 -3
  81. metadata +107 -677
@@ -1,9 +1,6 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Notifier
5
3
  class Nagios < Base
6
-
7
4
  ##
8
5
  # Host of Nagios server to notify on backup completion.
9
6
  attr_accessor :nagios_host
@@ -30,8 +27,8 @@ module Backup
30
27
 
31
28
  @nagios_host ||= Config.hostname
32
29
  @nagios_port ||= 5667
33
- @send_nsca_cfg||= "/etc/nagios/send_nsca.cfg"
34
- @service_name ||= "Backup #{ model.trigger }"
30
+ @send_nsca_cfg ||= "/etc/nagios/send_nsca.cfg"
31
+ @service_name ||= "Backup #{model.trigger}"
35
32
  @service_host ||= Config.hostname
36
33
  end
37
34
 
@@ -55,15 +52,14 @@ module Backup
55
52
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
56
53
  #
57
54
  def notify!(status)
58
- send_message(message.call(model, :status => status_data_for(status)))
55
+ send_message(message.call(model, status: status_data_for(status)))
59
56
  end
60
57
 
61
58
  def send_message(message)
62
- cmd = "#{ utility(:send_nsca) } -H '#{ nagios_host }' -p '#{ nagios_port }' -c '#{ send_nsca_cfg }'"
59
+ cmd = "#{utility(:send_nsca)} -H '#{nagios_host}' -p '#{nagios_port}' -c '#{send_nsca_cfg}'"
63
60
  msg = [service_host, service_name, model.exit_status, message].join("\t")
64
- run("echo '#{ msg }' | #{ cmd }")
61
+ run("echo '#{msg}' | #{cmd}")
65
62
  end
66
-
67
63
  end
68
64
  end
69
65
  end
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'pagerduty'
1
+ require "pagerduty"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class PagerDuty < Base
7
-
8
6
  ##
9
7
  # PagerDuty Service API Key. Should be a 32 character hex string.
10
8
  attr_accessor :service_key
@@ -46,15 +44,15 @@ module Backup
46
44
  incident_description = "Backup - #{model.label}"
47
45
  incident_key = "backup/#{model.trigger}"
48
46
  incident_details = {
49
- :incident_key => incident_key,
50
- :details => {
51
- :trigger => model.trigger,
52
- :label => model.label,
53
- :started_at => model.started_at,
54
- :finished_at => model.finished_at,
55
- :duration => model.duration,
56
- :status => status,
57
- :exception => model.exception
47
+ incident_key: incident_key,
48
+ details: {
49
+ trigger: model.trigger,
50
+ label: model.label,
51
+ started_at: model.started_at,
52
+ finished_at: model.finished_at,
53
+ duration: model.duration,
54
+ status: status,
55
+ exception: model.exception
58
56
  }
59
57
  }
60
58
 
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'uri'
1
+ require "uri"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Prowl < Base
7
-
8
6
  ##
9
7
  # Application name
10
8
  # Tell something like your server name. Example: "Server1 Backup"
@@ -16,10 +14,13 @@ module Backup
16
14
  attr_accessor :api_key
17
15
 
18
16
  def initialize(model, &block)
19
- @message = lambda do |model, data|
20
- "#{ model.label } (#{ model.trigger })"
21
- end
17
+ @message =
18
+ lambda do |m, _|
19
+ "#{m.label} (#{m.trigger})"
20
+ end
21
+
22
22
  super
23
+
23
24
  instance_eval(&block) if block_given?
24
25
  end
25
26
 
@@ -47,22 +48,21 @@ module Backup
47
48
  end
48
49
 
49
50
  def send_message(status)
50
- uri = 'https://api.prowlapp.com/publicapi/add'
51
+ uri = "https://api.prowlapp.com/publicapi/add"
51
52
  status_data = status_data_for(status)
52
53
  data = {
53
- :application => application,
54
- :apikey => api_key,
55
- :event => status_data[:message],
56
- :description => message.call(model, :status => status_data)
54
+ application: application,
55
+ apikey: api_key,
56
+ event: status_data[:message],
57
+ description: message.call(model, status: status_data)
57
58
  }
58
59
  options = {
59
- :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
60
- :body => URI.encode_www_form(data)
60
+ headers: { "Content-Type" => "application/x-www-form-urlencoded" },
61
+ body: URI.encode_www_form(data)
61
62
  }
62
- options.merge!(:expects => 200) # raise error if unsuccessful
63
+ options[:expects] = 200 # raise error if unsuccessful
63
64
  Excon.post(uri, options)
64
65
  end
65
-
66
66
  end
67
67
  end
68
68
  end
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'uri'
1
+ require "uri"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Pushover < Base
7
-
8
6
  ##
9
7
  # The API User Token
10
8
  attr_accessor :user
@@ -51,24 +49,23 @@ module Backup
51
49
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
52
50
  #
53
51
  def notify!(status)
54
- send_message(message.call(model, :status => status_data_for(status)))
52
+ send_message(message.call(model, status: status_data_for(status)))
55
53
  end
56
54
 
57
55
  def send_message(message)
58
- uri = 'https://api.pushover.net/1/messages.json'
59
- data = { :user => user, :token => token, :message => message }
56
+ uri = "https://api.pushover.net/1/messages.json"
57
+ data = { user: user, token: token, message: message }
60
58
  [:device, :title, :priority].each do |param|
61
59
  val = send(param)
62
60
  data.merge!(param => val) if val
63
61
  end
64
62
  options = {
65
- :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
66
- :body => URI.encode_www_form(data)
63
+ headers: { "Content-Type" => "application/x-www-form-urlencoded" },
64
+ body: URI.encode_www_form(data)
67
65
  }
68
- options.merge!(:expects => 200) # raise error if unsuccessful
66
+ options[:expects] = 200 # raise error if unsuccessful
69
67
  Excon.post(uri, options)
70
68
  end
71
-
72
69
  end
73
70
  end
74
71
  end
@@ -1,13 +1,12 @@
1
- # encoding: utf-8
2
- require 'aws/ses'
1
+ require "aws-sdk"
2
+ require "mail"
3
3
 
4
4
  module Backup
5
5
  module Notifier
6
6
  class Ses < Base
7
-
8
7
  ##
9
8
  # Amazon Simple Email Service (SES) Credentials
10
- attr_accessor :access_key_id, :secret_access_key
9
+ attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
11
10
 
12
11
  ##
13
12
  # SES Region
@@ -21,11 +20,23 @@ module Backup
21
20
  # Receiver Email Address
22
21
  attr_accessor :to
23
22
 
23
+ ##
24
+ # CC receiver Email Address
25
+ attr_accessor :cc
26
+
27
+ ##
28
+ # BCC receiver Email Address
29
+ attr_accessor :bcc
30
+
31
+ ##
32
+ # Set reply to email address
33
+ attr_accessor :reply_to
34
+
24
35
  def initialize(model, &block)
25
36
  super
26
37
  instance_eval(&block) if block_given?
27
38
 
28
- @region ||= 'eu-west-1'
39
+ @region ||= "eu-west-1"
29
40
  @send_log_on ||= [:warning, :failure]
30
41
  end
31
42
 
@@ -39,10 +50,15 @@ module Backup
39
50
  private
40
51
 
41
52
  def client
42
- AWS::SES::Base.new(
43
- :access_key_id => access_key_id,
44
- :secret_access_key => secret_access_key,
45
- :server => "email.#{region}.amazonaws.com"
53
+ credentials = if use_iam_profile
54
+ Aws::InstanceProfileCredentials.new
55
+ else
56
+ Aws::Credentials.new(access_key_id, secret_access_key)
57
+ end
58
+
59
+ Aws::SES::Client.new(
60
+ region: region,
61
+ credentials: credentials
46
62
  )
47
63
  end
48
64
 
@@ -66,22 +82,41 @@ module Backup
66
82
  # : backup log, if `on_failure` is `true`.
67
83
  #
68
84
  def notify!(status)
69
- email = ::Mail.new(:to => to, :from => from)
70
- email.subject = message.call(model, :status => status_data_for(status))
85
+ email = ::Mail.new
86
+ email.to = to
87
+ email.from = from
88
+ email.cc = cc
89
+ email.bcc = bcc
90
+ email.reply_to = reply_to
91
+ email.subject = message.call(model, status: status_data_for(status))
92
+
93
+ # By default, the `mail` gem doesn't include BCC in raw output, which is
94
+ # needed for SES to send to those addresses.
95
+ email[:bcc].include_in_headers = true
71
96
 
72
97
  send_log = send_log_on.include?(status)
73
- template = Backup::Template.new({ :model => model, :send_log => send_log })
74
- email.body = template.result('notifier/mail/%s.erb' % status.to_s)
98
+ template = Backup::Template.new(model: model, send_log: send_log)
99
+ email.body = template.result(sprintf("notifier/mail/%s.erb", status.to_s))
75
100
 
76
101
  if send_log
77
102
  email.convert_to_multipart
78
- email.attachments["#{ model.time }.#{ model.trigger }.log"] = {
79
- :mime_type => 'text/plain;',
80
- :content => Logger.messages.map(&:formatted_lines).flatten.join("\n")
103
+ email.attachments["#{model.time}.#{model.trigger}.log"] = {
104
+ mime_type: "text/plain;",
105
+ content: Logger.messages.map(&:formatted_lines).flatten.join("\n")
106
+ }
107
+ end
108
+
109
+ send_opts = {
110
+ raw_message: {
111
+ data: email.to_s
81
112
  }
113
+ }
114
+
115
+ if email.respond_to?(:destinations)
116
+ send_opts[:destinations] = email.destinations
82
117
  end
83
118
 
84
- client.send_raw_email(email)
119
+ client.send_raw_email(send_opts)
85
120
  end
86
121
  end
87
122
  end
@@ -1,11 +1,9 @@
1
- # encoding: utf-8
2
- require 'uri'
3
- require 'json'
1
+ require "uri"
2
+ require "json"
4
3
 
5
4
  module Backup
6
5
  module Notifier
7
6
  class Slack < Base
8
-
9
7
  ##
10
8
  # The incoming webhook url
11
9
  attr_accessor :webhook_url
@@ -38,7 +36,7 @@ module Backup
38
36
  instance_eval(&block) if block_given?
39
37
 
40
38
  @send_log_on ||= [:warning, :failure]
41
- @icon_emoji ||= ':floppy_disk:'
39
+ @icon_emoji ||= ":floppy_disk:"
42
40
  end
43
41
 
44
42
  private
@@ -62,8 +60,8 @@ module Backup
62
60
  #
63
61
  def notify!(status)
64
62
  data = {
65
- :text => message.call(model, :status => status_data_for(status)),
66
- :attachments => [attachment(status)]
63
+ text: message.call(model, status: status_data_for(status)),
64
+ attachments: [attachment(status)]
67
65
  }
68
66
  [:channel, :username, :icon_emoji].each do |param|
69
67
  val = send(param)
@@ -71,43 +69,43 @@ module Backup
71
69
  end
72
70
 
73
71
  options = {
74
- :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
75
- :body => URI.encode_www_form(:payload => JSON.dump(data))
72
+ headers: { "Content-Type" => "application/x-www-form-urlencoded" },
73
+ body: URI.encode_www_form(payload: JSON.dump(data))
76
74
  }
77
- options.merge!(:expects => 200) # raise error if unsuccessful
75
+ options[:expects] = 200 # raise error if unsuccessful
78
76
  Excon.post(uri, options)
79
77
  end
80
78
 
81
79
  def attachment(status)
82
80
  {
83
- :fallback => "#{title(status)} - Job: #{model.label} (#{model.trigger})",
84
- :text => title(status),
85
- :color => color(status),
86
- :fields => [
81
+ fallback: "#{title(status)} - Job: #{model.label} (#{model.trigger})",
82
+ text: title(status),
83
+ color: color(status),
84
+ fields: [
87
85
  {
88
- :title => "Job",
89
- :value => "#{model.label} (#{model.trigger})",
90
- :short => false
86
+ title: "Job",
87
+ value: "#{model.label} (#{model.trigger})",
88
+ short: false
91
89
  },
92
90
  {
93
- :title => "Started",
94
- :value => model.started_at,
95
- :short => true
91
+ title: "Started",
92
+ value: model.started_at,
93
+ short: true
96
94
  },
97
95
  {
98
- :title => "Finished",
99
- :value => model.finished_at,
100
- :short => true
96
+ title: "Finished",
97
+ value: model.finished_at,
98
+ short: true
101
99
  },
102
100
  {
103
- :title => "Duration",
104
- :value => model.duration,
105
- :short => true
101
+ title: "Duration",
102
+ value: model.duration,
103
+ short: true
106
104
  },
107
105
  {
108
- :title => "Version",
109
- :value => "Backup v#{Backup::VERSION}\nRuby: #{RUBY_DESCRIPTION}",
110
- :short => false
106
+ title: "Version",
107
+ value: "Backup v#{Backup::VERSION}\nRuby: #{RUBY_DESCRIPTION}",
108
+ short: false
111
109
  },
112
110
  log_field(status)
113
111
  ].compact
@@ -116,27 +114,28 @@ module Backup
116
114
 
117
115
  def log_field(status)
118
116
  send_log = send_log_on.include?(status)
117
+ return unless send_log
119
118
 
120
- return {
121
- :title => "Detailed Backup Log",
122
- :value => Logger.messages.map(&:formatted_lines).flatten.join("\n"),
123
- :short => false,
124
- } if send_log
119
+ {
120
+ title: "Detailed Backup Log",
121
+ value: Logger.messages.map(&:formatted_lines).flatten.join("\n"),
122
+ short: false
123
+ }
125
124
  end
126
125
 
127
126
  def color(status)
128
127
  case status
129
- when :success then 'good'
130
- when :failure then 'danger'
131
- when :warning then 'warning'
128
+ when :success then "good"
129
+ when :failure then "danger"
130
+ when :warning then "warning"
132
131
  end
133
132
  end
134
133
 
135
134
  def title(status)
136
135
  case status
137
- when :success then 'Backup Completed Successfully!'
138
- when :failure then 'Backup Failed!'
139
- when :warning then 'Backup Completed Successfully (with Warnings)!'
136
+ when :success then "Backup Completed Successfully!"
137
+ when :failure then "Backup Failed!"
138
+ when :warning then "Backup Completed Successfully (with Warnings)!"
140
139
  end
141
140
  end
142
141
 
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'twitter'
1
+ require "twitter"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Twitter < Base
7
-
8
6
  ##
9
7
  # Twitter consumer key credentials
10
8
  attr_accessor :consumer_key, :consumer_secret
@@ -38,7 +36,7 @@ module Backup
38
36
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
39
37
  #
40
38
  def notify!(status)
41
- send_message(message.call(model, :status => status_data_for(status)))
39
+ send_message(message.call(model, status: status_data_for(status)))
42
40
  end
43
41
 
44
42
  # Twitter::Client will raise an error if unsuccessful.
@@ -52,7 +50,6 @@ module Backup
52
50
 
53
51
  client.update(message)
54
52
  end
55
-
56
53
  end
57
54
  end
58
55
  end