backup 4.3.0 → 5.0.0.beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +5 -5
  2. data/LICENSE +19 -0
  3. data/README.md +13 -9
  4. data/bin/docker_test +24 -0
  5. data/lib/backup.rb +74 -78
  6. data/lib/backup/archive.rb +31 -32
  7. data/lib/backup/binder.rb +2 -6
  8. data/lib/backup/cleaner.rb +14 -18
  9. data/lib/backup/cli.rb +104 -108
  10. data/lib/backup/cloud_io/base.rb +4 -7
  11. data/lib/backup/cloud_io/cloud_files.rb +60 -62
  12. data/lib/backup/cloud_io/s3.rb +69 -76
  13. data/lib/backup/compressor/base.rb +4 -7
  14. data/lib/backup/compressor/bzip2.rb +3 -7
  15. data/lib/backup/compressor/custom.rb +2 -6
  16. data/lib/backup/compressor/gzip.rb +16 -17
  17. data/lib/backup/config.rb +17 -18
  18. data/lib/backup/config/dsl.rb +16 -17
  19. data/lib/backup/config/helpers.rb +10 -16
  20. data/lib/backup/database/base.rb +22 -21
  21. data/lib/backup/database/mongodb.rb +36 -37
  22. data/lib/backup/database/mysql.rb +40 -41
  23. data/lib/backup/database/openldap.rb +8 -10
  24. data/lib/backup/database/postgresql.rb +29 -30
  25. data/lib/backup/database/redis.rb +27 -30
  26. data/lib/backup/database/riak.rb +15 -18
  27. data/lib/backup/database/sqlite.rb +4 -6
  28. data/lib/backup/encryptor/base.rb +2 -4
  29. data/lib/backup/encryptor/gpg.rb +49 -59
  30. data/lib/backup/encryptor/open_ssl.rb +11 -14
  31. data/lib/backup/errors.rb +7 -12
  32. data/lib/backup/logger.rb +16 -18
  33. data/lib/backup/logger/console.rb +5 -8
  34. data/lib/backup/logger/fog_adapter.rb +2 -6
  35. data/lib/backup/logger/logfile.rb +10 -12
  36. data/lib/backup/logger/syslog.rb +2 -4
  37. data/lib/backup/model.rb +33 -40
  38. data/lib/backup/notifier/base.rb +24 -26
  39. data/lib/backup/notifier/campfire.rb +9 -11
  40. data/lib/backup/notifier/command.rb +3 -3
  41. data/lib/backup/notifier/datadog.rb +9 -12
  42. data/lib/backup/notifier/flowdock.rb +13 -17
  43. data/lib/backup/notifier/hipchat.rb +18 -14
  44. data/lib/backup/notifier/http_post.rb +11 -14
  45. data/lib/backup/notifier/mail.rb +42 -54
  46. data/lib/backup/notifier/nagios.rb +5 -9
  47. data/lib/backup/notifier/pagerduty.rb +10 -12
  48. data/lib/backup/notifier/prowl.rb +15 -15
  49. data/lib/backup/notifier/pushover.rb +7 -10
  50. data/lib/backup/notifier/ses.rb +52 -17
  51. data/lib/backup/notifier/slack.rb +39 -40
  52. data/lib/backup/notifier/twitter.rb +2 -5
  53. data/lib/backup/notifier/zabbix.rb +11 -14
  54. data/lib/backup/package.rb +5 -9
  55. data/lib/backup/packager.rb +16 -17
  56. data/lib/backup/pipeline.rb +17 -21
  57. data/lib/backup/splitter.rb +8 -11
  58. data/lib/backup/storage/base.rb +5 -8
  59. data/lib/backup/storage/cloud_files.rb +21 -23
  60. data/lib/backup/storage/cycler.rb +10 -15
  61. data/lib/backup/storage/dropbox.rb +15 -21
  62. data/lib/backup/storage/ftp.rb +14 -10
  63. data/lib/backup/storage/local.rb +5 -8
  64. data/lib/backup/storage/qiniu.rb +8 -8
  65. data/lib/backup/storage/rsync.rb +24 -26
  66. data/lib/backup/storage/s3.rb +27 -28
  67. data/lib/backup/storage/scp.rb +10 -12
  68. data/lib/backup/storage/sftp.rb +10 -12
  69. data/lib/backup/syncer/base.rb +5 -8
  70. data/lib/backup/syncer/cloud/base.rb +27 -30
  71. data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
  72. data/lib/backup/syncer/cloud/local_file.rb +5 -8
  73. data/lib/backup/syncer/cloud/s3.rb +23 -24
  74. data/lib/backup/syncer/rsync/base.rb +6 -10
  75. data/lib/backup/syncer/rsync/local.rb +1 -5
  76. data/lib/backup/syncer/rsync/pull.rb +6 -10
  77. data/lib/backup/syncer/rsync/push.rb +18 -22
  78. data/lib/backup/template.rb +9 -14
  79. data/lib/backup/utilities.rb +78 -69
  80. data/lib/backup/version.rb +1 -3
  81. metadata +107 -677
@@ -1,9 +1,6 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Notifier
5
3
  class Nagios < Base
6
-
7
4
  ##
8
5
  # Host of Nagios server to notify on backup completion.
9
6
  attr_accessor :nagios_host
@@ -30,8 +27,8 @@ module Backup
30
27
 
31
28
  @nagios_host ||= Config.hostname
32
29
  @nagios_port ||= 5667
33
- @send_nsca_cfg||= "/etc/nagios/send_nsca.cfg"
34
- @service_name ||= "Backup #{ model.trigger }"
30
+ @send_nsca_cfg ||= "/etc/nagios/send_nsca.cfg"
31
+ @service_name ||= "Backup #{model.trigger}"
35
32
  @service_host ||= Config.hostname
36
33
  end
37
34
 
@@ -55,15 +52,14 @@ module Backup
55
52
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
56
53
  #
57
54
  def notify!(status)
58
- send_message(message.call(model, :status => status_data_for(status)))
55
+ send_message(message.call(model, status: status_data_for(status)))
59
56
  end
60
57
 
61
58
  def send_message(message)
62
- cmd = "#{ utility(:send_nsca) } -H '#{ nagios_host }' -p '#{ nagios_port }' -c '#{ send_nsca_cfg }'"
59
+ cmd = "#{utility(:send_nsca)} -H '#{nagios_host}' -p '#{nagios_port}' -c '#{send_nsca_cfg}'"
63
60
  msg = [service_host, service_name, model.exit_status, message].join("\t")
64
- run("echo '#{ msg }' | #{ cmd }")
61
+ run("echo '#{msg}' | #{cmd}")
65
62
  end
66
-
67
63
  end
68
64
  end
69
65
  end
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'pagerduty'
1
+ require "pagerduty"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class PagerDuty < Base
7
-
8
6
  ##
9
7
  # PagerDuty Service API Key. Should be a 32 character hex string.
10
8
  attr_accessor :service_key
@@ -46,15 +44,15 @@ module Backup
46
44
  incident_description = "Backup - #{model.label}"
47
45
  incident_key = "backup/#{model.trigger}"
48
46
  incident_details = {
49
- :incident_key => incident_key,
50
- :details => {
51
- :trigger => model.trigger,
52
- :label => model.label,
53
- :started_at => model.started_at,
54
- :finished_at => model.finished_at,
55
- :duration => model.duration,
56
- :status => status,
57
- :exception => model.exception
47
+ incident_key: incident_key,
48
+ details: {
49
+ trigger: model.trigger,
50
+ label: model.label,
51
+ started_at: model.started_at,
52
+ finished_at: model.finished_at,
53
+ duration: model.duration,
54
+ status: status,
55
+ exception: model.exception
58
56
  }
59
57
  }
60
58
 
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'uri'
1
+ require "uri"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Prowl < Base
7
-
8
6
  ##
9
7
  # Application name
10
8
  # Tell something like your server name. Example: "Server1 Backup"
@@ -16,10 +14,13 @@ module Backup
16
14
  attr_accessor :api_key
17
15
 
18
16
  def initialize(model, &block)
19
- @message = lambda do |model, data|
20
- "#{ model.label } (#{ model.trigger })"
21
- end
17
+ @message =
18
+ lambda do |m, _|
19
+ "#{m.label} (#{m.trigger})"
20
+ end
21
+
22
22
  super
23
+
23
24
  instance_eval(&block) if block_given?
24
25
  end
25
26
 
@@ -47,22 +48,21 @@ module Backup
47
48
  end
48
49
 
49
50
  def send_message(status)
50
- uri = 'https://api.prowlapp.com/publicapi/add'
51
+ uri = "https://api.prowlapp.com/publicapi/add"
51
52
  status_data = status_data_for(status)
52
53
  data = {
53
- :application => application,
54
- :apikey => api_key,
55
- :event => status_data[:message],
56
- :description => message.call(model, :status => status_data)
54
+ application: application,
55
+ apikey: api_key,
56
+ event: status_data[:message],
57
+ description: message.call(model, status: status_data)
57
58
  }
58
59
  options = {
59
- :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
60
- :body => URI.encode_www_form(data)
60
+ headers: { "Content-Type" => "application/x-www-form-urlencoded" },
61
+ body: URI.encode_www_form(data)
61
62
  }
62
- options.merge!(:expects => 200) # raise error if unsuccessful
63
+ options[:expects] = 200 # raise error if unsuccessful
63
64
  Excon.post(uri, options)
64
65
  end
65
-
66
66
  end
67
67
  end
68
68
  end
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'uri'
1
+ require "uri"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Pushover < Base
7
-
8
6
  ##
9
7
  # The API User Token
10
8
  attr_accessor :user
@@ -51,24 +49,23 @@ module Backup
51
49
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
52
50
  #
53
51
  def notify!(status)
54
- send_message(message.call(model, :status => status_data_for(status)))
52
+ send_message(message.call(model, status: status_data_for(status)))
55
53
  end
56
54
 
57
55
  def send_message(message)
58
- uri = 'https://api.pushover.net/1/messages.json'
59
- data = { :user => user, :token => token, :message => message }
56
+ uri = "https://api.pushover.net/1/messages.json"
57
+ data = { user: user, token: token, message: message }
60
58
  [:device, :title, :priority].each do |param|
61
59
  val = send(param)
62
60
  data.merge!(param => val) if val
63
61
  end
64
62
  options = {
65
- :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
66
- :body => URI.encode_www_form(data)
63
+ headers: { "Content-Type" => "application/x-www-form-urlencoded" },
64
+ body: URI.encode_www_form(data)
67
65
  }
68
- options.merge!(:expects => 200) # raise error if unsuccessful
66
+ options[:expects] = 200 # raise error if unsuccessful
69
67
  Excon.post(uri, options)
70
68
  end
71
-
72
69
  end
73
70
  end
74
71
  end
@@ -1,13 +1,12 @@
1
- # encoding: utf-8
2
- require 'aws/ses'
1
+ require "aws-sdk"
2
+ require "mail"
3
3
 
4
4
  module Backup
5
5
  module Notifier
6
6
  class Ses < Base
7
-
8
7
  ##
9
8
  # Amazon Simple Email Service (SES) Credentials
10
- attr_accessor :access_key_id, :secret_access_key
9
+ attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
11
10
 
12
11
  ##
13
12
  # SES Region
@@ -21,11 +20,23 @@ module Backup
21
20
  # Receiver Email Address
22
21
  attr_accessor :to
23
22
 
23
+ ##
24
+ # CC receiver Email Address
25
+ attr_accessor :cc
26
+
27
+ ##
28
+ # BCC receiver Email Address
29
+ attr_accessor :bcc
30
+
31
+ ##
32
+ # Set reply to email address
33
+ attr_accessor :reply_to
34
+
24
35
  def initialize(model, &block)
25
36
  super
26
37
  instance_eval(&block) if block_given?
27
38
 
28
- @region ||= 'eu-west-1'
39
+ @region ||= "eu-west-1"
29
40
  @send_log_on ||= [:warning, :failure]
30
41
  end
31
42
 
@@ -39,10 +50,15 @@ module Backup
39
50
  private
40
51
 
41
52
  def client
42
- AWS::SES::Base.new(
43
- :access_key_id => access_key_id,
44
- :secret_access_key => secret_access_key,
45
- :server => "email.#{region}.amazonaws.com"
53
+ credentials = if use_iam_profile
54
+ Aws::InstanceProfileCredentials.new
55
+ else
56
+ Aws::Credentials.new(access_key_id, secret_access_key)
57
+ end
58
+
59
+ Aws::SES::Client.new(
60
+ region: region,
61
+ credentials: credentials
46
62
  )
47
63
  end
48
64
 
@@ -66,22 +82,41 @@ module Backup
66
82
  # : backup log, if `on_failure` is `true`.
67
83
  #
68
84
  def notify!(status)
69
- email = ::Mail.new(:to => to, :from => from)
70
- email.subject = message.call(model, :status => status_data_for(status))
85
+ email = ::Mail.new
86
+ email.to = to
87
+ email.from = from
88
+ email.cc = cc
89
+ email.bcc = bcc
90
+ email.reply_to = reply_to
91
+ email.subject = message.call(model, status: status_data_for(status))
92
+
93
+ # By default, the `mail` gem doesn't include BCC in raw output, which is
94
+ # needed for SES to send to those addresses.
95
+ email[:bcc].include_in_headers = true
71
96
 
72
97
  send_log = send_log_on.include?(status)
73
- template = Backup::Template.new({ :model => model, :send_log => send_log })
74
- email.body = template.result('notifier/mail/%s.erb' % status.to_s)
98
+ template = Backup::Template.new(model: model, send_log: send_log)
99
+ email.body = template.result(sprintf("notifier/mail/%s.erb", status.to_s))
75
100
 
76
101
  if send_log
77
102
  email.convert_to_multipart
78
- email.attachments["#{ model.time }.#{ model.trigger }.log"] = {
79
- :mime_type => 'text/plain;',
80
- :content => Logger.messages.map(&:formatted_lines).flatten.join("\n")
103
+ email.attachments["#{model.time}.#{model.trigger}.log"] = {
104
+ mime_type: "text/plain;",
105
+ content: Logger.messages.map(&:formatted_lines).flatten.join("\n")
106
+ }
107
+ end
108
+
109
+ send_opts = {
110
+ raw_message: {
111
+ data: email.to_s
81
112
  }
113
+ }
114
+
115
+ if email.respond_to?(:destinations)
116
+ send_opts[:destinations] = email.destinations
82
117
  end
83
118
 
84
- client.send_raw_email(email)
119
+ client.send_raw_email(send_opts)
85
120
  end
86
121
  end
87
122
  end
@@ -1,11 +1,9 @@
1
- # encoding: utf-8
2
- require 'uri'
3
- require 'json'
1
+ require "uri"
2
+ require "json"
4
3
 
5
4
  module Backup
6
5
  module Notifier
7
6
  class Slack < Base
8
-
9
7
  ##
10
8
  # The incoming webhook url
11
9
  attr_accessor :webhook_url
@@ -38,7 +36,7 @@ module Backup
38
36
  instance_eval(&block) if block_given?
39
37
 
40
38
  @send_log_on ||= [:warning, :failure]
41
- @icon_emoji ||= ':floppy_disk:'
39
+ @icon_emoji ||= ":floppy_disk:"
42
40
  end
43
41
 
44
42
  private
@@ -62,8 +60,8 @@ module Backup
62
60
  #
63
61
  def notify!(status)
64
62
  data = {
65
- :text => message.call(model, :status => status_data_for(status)),
66
- :attachments => [attachment(status)]
63
+ text: message.call(model, status: status_data_for(status)),
64
+ attachments: [attachment(status)]
67
65
  }
68
66
  [:channel, :username, :icon_emoji].each do |param|
69
67
  val = send(param)
@@ -71,43 +69,43 @@ module Backup
71
69
  end
72
70
 
73
71
  options = {
74
- :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
75
- :body => URI.encode_www_form(:payload => JSON.dump(data))
72
+ headers: { "Content-Type" => "application/x-www-form-urlencoded" },
73
+ body: URI.encode_www_form(payload: JSON.dump(data))
76
74
  }
77
- options.merge!(:expects => 200) # raise error if unsuccessful
75
+ options[:expects] = 200 # raise error if unsuccessful
78
76
  Excon.post(uri, options)
79
77
  end
80
78
 
81
79
  def attachment(status)
82
80
  {
83
- :fallback => "#{title(status)} - Job: #{model.label} (#{model.trigger})",
84
- :text => title(status),
85
- :color => color(status),
86
- :fields => [
81
+ fallback: "#{title(status)} - Job: #{model.label} (#{model.trigger})",
82
+ text: title(status),
83
+ color: color(status),
84
+ fields: [
87
85
  {
88
- :title => "Job",
89
- :value => "#{model.label} (#{model.trigger})",
90
- :short => false
86
+ title: "Job",
87
+ value: "#{model.label} (#{model.trigger})",
88
+ short: false
91
89
  },
92
90
  {
93
- :title => "Started",
94
- :value => model.started_at,
95
- :short => true
91
+ title: "Started",
92
+ value: model.started_at,
93
+ short: true
96
94
  },
97
95
  {
98
- :title => "Finished",
99
- :value => model.finished_at,
100
- :short => true
96
+ title: "Finished",
97
+ value: model.finished_at,
98
+ short: true
101
99
  },
102
100
  {
103
- :title => "Duration",
104
- :value => model.duration,
105
- :short => true
101
+ title: "Duration",
102
+ value: model.duration,
103
+ short: true
106
104
  },
107
105
  {
108
- :title => "Version",
109
- :value => "Backup v#{Backup::VERSION}\nRuby: #{RUBY_DESCRIPTION}",
110
- :short => false
106
+ title: "Version",
107
+ value: "Backup v#{Backup::VERSION}\nRuby: #{RUBY_DESCRIPTION}",
108
+ short: false
111
109
  },
112
110
  log_field(status)
113
111
  ].compact
@@ -116,27 +114,28 @@ module Backup
116
114
 
117
115
  def log_field(status)
118
116
  send_log = send_log_on.include?(status)
117
+ return unless send_log
119
118
 
120
- return {
121
- :title => "Detailed Backup Log",
122
- :value => Logger.messages.map(&:formatted_lines).flatten.join("\n"),
123
- :short => false,
124
- } if send_log
119
+ {
120
+ title: "Detailed Backup Log",
121
+ value: Logger.messages.map(&:formatted_lines).flatten.join("\n"),
122
+ short: false
123
+ }
125
124
  end
126
125
 
127
126
  def color(status)
128
127
  case status
129
- when :success then 'good'
130
- when :failure then 'danger'
131
- when :warning then 'warning'
128
+ when :success then "good"
129
+ when :failure then "danger"
130
+ when :warning then "warning"
132
131
  end
133
132
  end
134
133
 
135
134
  def title(status)
136
135
  case status
137
- when :success then 'Backup Completed Successfully!'
138
- when :failure then 'Backup Failed!'
139
- when :warning then 'Backup Completed Successfully (with Warnings)!'
136
+ when :success then "Backup Completed Successfully!"
137
+ when :failure then "Backup Failed!"
138
+ when :warning then "Backup Completed Successfully (with Warnings)!"
140
139
  end
141
140
  end
142
141
 
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'twitter'
1
+ require "twitter"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Twitter < Base
7
-
8
6
  ##
9
7
  # Twitter consumer key credentials
10
8
  attr_accessor :consumer_key, :consumer_secret
@@ -38,7 +36,7 @@ module Backup
38
36
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
39
37
  #
40
38
  def notify!(status)
41
- send_message(message.call(model, :status => status_data_for(status)))
39
+ send_message(message.call(model, status: status_data_for(status)))
42
40
  end
43
41
 
44
42
  # Twitter::Client will raise an error if unsuccessful.
@@ -52,7 +50,6 @@ module Backup
52
50
 
53
51
  client.update(message)
54
52
  end
55
-
56
53
  end
57
54
  end
58
55
  end