backup 4.4.1 → 5.0.0.beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +5 -5
  2. data/LICENSE +19 -0
  3. data/README.md +1 -1
  4. data/lib/backup.rb +74 -78
  5. data/lib/backup/archive.rb +31 -32
  6. data/lib/backup/binder.rb +2 -6
  7. data/lib/backup/cleaner.rb +14 -18
  8. data/lib/backup/cli.rb +104 -108
  9. data/lib/backup/cloud_io/base.rb +4 -7
  10. data/lib/backup/cloud_io/cloud_files.rb +60 -62
  11. data/lib/backup/cloud_io/s3.rb +69 -76
  12. data/lib/backup/compressor/base.rb +4 -7
  13. data/lib/backup/compressor/bzip2.rb +3 -7
  14. data/lib/backup/compressor/custom.rb +2 -6
  15. data/lib/backup/compressor/gzip.rb +16 -17
  16. data/lib/backup/config.rb +17 -18
  17. data/lib/backup/config/dsl.rb +16 -17
  18. data/lib/backup/config/helpers.rb +10 -16
  19. data/lib/backup/database/base.rb +22 -21
  20. data/lib/backup/database/mongodb.rb +36 -37
  21. data/lib/backup/database/mysql.rb +40 -41
  22. data/lib/backup/database/openldap.rb +8 -10
  23. data/lib/backup/database/postgresql.rb +29 -30
  24. data/lib/backup/database/redis.rb +27 -30
  25. data/lib/backup/database/riak.rb +15 -18
  26. data/lib/backup/database/sqlite.rb +4 -6
  27. data/lib/backup/encryptor/base.rb +2 -4
  28. data/lib/backup/encryptor/gpg.rb +49 -59
  29. data/lib/backup/encryptor/open_ssl.rb +11 -14
  30. data/lib/backup/errors.rb +7 -12
  31. data/lib/backup/logger.rb +16 -18
  32. data/lib/backup/logger/console.rb +5 -8
  33. data/lib/backup/logger/fog_adapter.rb +2 -6
  34. data/lib/backup/logger/logfile.rb +10 -12
  35. data/lib/backup/logger/syslog.rb +2 -4
  36. data/lib/backup/model.rb +75 -40
  37. data/lib/backup/notifier/base.rb +24 -26
  38. data/lib/backup/notifier/campfire.rb +9 -11
  39. data/lib/backup/notifier/command.rb +0 -3
  40. data/lib/backup/notifier/datadog.rb +9 -12
  41. data/lib/backup/notifier/flowdock.rb +13 -17
  42. data/lib/backup/notifier/hipchat.rb +11 -13
  43. data/lib/backup/notifier/http_post.rb +11 -14
  44. data/lib/backup/notifier/mail.rb +44 -47
  45. data/lib/backup/notifier/nagios.rb +5 -9
  46. data/lib/backup/notifier/pagerduty.rb +10 -12
  47. data/lib/backup/notifier/prowl.rb +15 -15
  48. data/lib/backup/notifier/pushover.rb +7 -10
  49. data/lib/backup/notifier/ses.rb +34 -16
  50. data/lib/backup/notifier/slack.rb +39 -40
  51. data/lib/backup/notifier/twitter.rb +2 -5
  52. data/lib/backup/notifier/zabbix.rb +11 -14
  53. data/lib/backup/package.rb +5 -9
  54. data/lib/backup/packager.rb +16 -17
  55. data/lib/backup/pipeline.rb +17 -21
  56. data/lib/backup/splitter.rb +8 -11
  57. data/lib/backup/storage/base.rb +5 -8
  58. data/lib/backup/storage/cloud_files.rb +21 -23
  59. data/lib/backup/storage/cycler.rb +10 -15
  60. data/lib/backup/storage/dropbox.rb +15 -21
  61. data/lib/backup/storage/ftp.rb +8 -10
  62. data/lib/backup/storage/local.rb +5 -8
  63. data/lib/backup/storage/qiniu.rb +8 -8
  64. data/lib/backup/storage/rsync.rb +24 -26
  65. data/lib/backup/storage/s3.rb +27 -28
  66. data/lib/backup/storage/scp.rb +10 -12
  67. data/lib/backup/storage/sftp.rb +10 -12
  68. data/lib/backup/syncer/base.rb +5 -8
  69. data/lib/backup/syncer/cloud/base.rb +27 -30
  70. data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
  71. data/lib/backup/syncer/cloud/local_file.rb +5 -8
  72. data/lib/backup/syncer/cloud/s3.rb +23 -24
  73. data/lib/backup/syncer/rsync/base.rb +6 -10
  74. data/lib/backup/syncer/rsync/local.rb +1 -5
  75. data/lib/backup/syncer/rsync/pull.rb +6 -10
  76. data/lib/backup/syncer/rsync/push.rb +18 -22
  77. data/lib/backup/template.rb +9 -14
  78. data/lib/backup/utilities.rb +82 -69
  79. data/lib/backup/version.rb +1 -3
  80. metadata +100 -660
@@ -1,9 +1,6 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Notifier
5
3
  class Nagios < Base
6
-
7
4
  ##
8
5
  # Host of Nagios server to notify on backup completion.
9
6
  attr_accessor :nagios_host
@@ -30,8 +27,8 @@ module Backup
30
27
 
31
28
  @nagios_host ||= Config.hostname
32
29
  @nagios_port ||= 5667
33
- @send_nsca_cfg||= "/etc/nagios/send_nsca.cfg"
34
- @service_name ||= "Backup #{ model.trigger }"
30
+ @send_nsca_cfg ||= "/etc/nagios/send_nsca.cfg"
31
+ @service_name ||= "Backup #{model.trigger}"
35
32
  @service_host ||= Config.hostname
36
33
  end
37
34
 
@@ -55,15 +52,14 @@ module Backup
55
52
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
56
53
  #
57
54
  def notify!(status)
58
- send_message(message.call(model, :status => status_data_for(status)))
55
+ send_message(message.call(model, status: status_data_for(status)))
59
56
  end
60
57
 
61
58
  def send_message(message)
62
- cmd = "#{ utility(:send_nsca) } -H '#{ nagios_host }' -p '#{ nagios_port }' -c '#{ send_nsca_cfg }'"
59
+ cmd = "#{utility(:send_nsca)} -H '#{nagios_host}' -p '#{nagios_port}' -c '#{send_nsca_cfg}'"
63
60
  msg = [service_host, service_name, model.exit_status, message].join("\t")
64
- run("echo '#{ msg }' | #{ cmd }")
61
+ run("echo '#{msg}' | #{cmd}")
65
62
  end
66
-
67
63
  end
68
64
  end
69
65
  end
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'pagerduty'
1
+ require "pagerduty"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class PagerDuty < Base
7
-
8
6
  ##
9
7
  # PagerDuty Service API Key. Should be a 32 character hex string.
10
8
  attr_accessor :service_key
@@ -46,15 +44,15 @@ module Backup
46
44
  incident_description = "Backup - #{model.label}"
47
45
  incident_key = "backup/#{model.trigger}"
48
46
  incident_details = {
49
- :incident_key => incident_key,
50
- :details => {
51
- :trigger => model.trigger,
52
- :label => model.label,
53
- :started_at => model.started_at,
54
- :finished_at => model.finished_at,
55
- :duration => model.duration,
56
- :status => status,
57
- :exception => model.exception
47
+ incident_key: incident_key,
48
+ details: {
49
+ trigger: model.trigger,
50
+ label: model.label,
51
+ started_at: model.started_at,
52
+ finished_at: model.finished_at,
53
+ duration: model.duration,
54
+ status: status,
55
+ exception: model.exception
58
56
  }
59
57
  }
60
58
 
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'uri'
1
+ require "uri"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Prowl < Base
7
-
8
6
  ##
9
7
  # Application name
10
8
  # Tell something like your server name. Example: "Server1 Backup"
@@ -16,10 +14,13 @@ module Backup
16
14
  attr_accessor :api_key
17
15
 
18
16
  def initialize(model, &block)
19
- @message = lambda do |model, data|
20
- "#{ model.label } (#{ model.trigger })"
21
- end
17
+ @message =
18
+ lambda do |m, _|
19
+ "#{m.label} (#{m.trigger})"
20
+ end
21
+
22
22
  super
23
+
23
24
  instance_eval(&block) if block_given?
24
25
  end
25
26
 
@@ -47,22 +48,21 @@ module Backup
47
48
  end
48
49
 
49
50
  def send_message(status)
50
- uri = 'https://api.prowlapp.com/publicapi/add'
51
+ uri = "https://api.prowlapp.com/publicapi/add"
51
52
  status_data = status_data_for(status)
52
53
  data = {
53
- :application => application,
54
- :apikey => api_key,
55
- :event => status_data[:message],
56
- :description => message.call(model, :status => status_data)
54
+ application: application,
55
+ apikey: api_key,
56
+ event: status_data[:message],
57
+ description: message.call(model, status: status_data)
57
58
  }
58
59
  options = {
59
- :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
60
- :body => URI.encode_www_form(data)
60
+ headers: { "Content-Type" => "application/x-www-form-urlencoded" },
61
+ body: URI.encode_www_form(data)
61
62
  }
62
- options.merge!(:expects => 200) # raise error if unsuccessful
63
+ options[:expects] = 200 # raise error if unsuccessful
63
64
  Excon.post(uri, options)
64
65
  end
65
-
66
66
  end
67
67
  end
68
68
  end
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'uri'
1
+ require "uri"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Pushover < Base
7
-
8
6
  ##
9
7
  # The API User Token
10
8
  attr_accessor :user
@@ -51,24 +49,23 @@ module Backup
51
49
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
52
50
  #
53
51
  def notify!(status)
54
- send_message(message.call(model, :status => status_data_for(status)))
52
+ send_message(message.call(model, status: status_data_for(status)))
55
53
  end
56
54
 
57
55
  def send_message(message)
58
- uri = 'https://api.pushover.net/1/messages.json'
59
- data = { :user => user, :token => token, :message => message }
56
+ uri = "https://api.pushover.net/1/messages.json"
57
+ data = { user: user, token: token, message: message }
60
58
  [:device, :title, :priority].each do |param|
61
59
  val = send(param)
62
60
  data.merge!(param => val) if val
63
61
  end
64
62
  options = {
65
- :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
66
- :body => URI.encode_www_form(data)
63
+ headers: { "Content-Type" => "application/x-www-form-urlencoded" },
64
+ body: URI.encode_www_form(data)
67
65
  }
68
- options.merge!(:expects => 200) # raise error if unsuccessful
66
+ options[:expects] = 200 # raise error if unsuccessful
69
67
  Excon.post(uri, options)
70
68
  end
71
-
72
69
  end
73
70
  end
74
71
  end
@@ -1,13 +1,12 @@
1
- # encoding: utf-8
2
- require 'aws/ses'
1
+ require "aws-sdk"
2
+ require "mail"
3
3
 
4
4
  module Backup
5
5
  module Notifier
6
6
  class Ses < Base
7
-
8
7
  ##
9
8
  # Amazon Simple Email Service (SES) Credentials
10
- attr_accessor :access_key_id, :secret_access_key
9
+ attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
11
10
 
12
11
  ##
13
12
  # SES Region
@@ -37,7 +36,7 @@ module Backup
37
36
  super
38
37
  instance_eval(&block) if block_given?
39
38
 
40
- @region ||= 'eu-west-1'
39
+ @region ||= "eu-west-1"
41
40
  @send_log_on ||= [:warning, :failure]
42
41
  end
43
42
 
@@ -51,10 +50,15 @@ module Backup
51
50
  private
52
51
 
53
52
  def client
54
- AWS::SES::Base.new(
55
- :access_key_id => access_key_id,
56
- :secret_access_key => secret_access_key,
57
- :server => "email.#{region}.amazonaws.com"
53
+ credentials = if use_iam_profile
54
+ Aws::InstanceProfileCredentials.new
55
+ else
56
+ Aws::Credentials.new(access_key_id, secret_access_key)
57
+ end
58
+
59
+ Aws::SES::Client.new(
60
+ region: region,
61
+ credentials: credentials
58
62
  )
59
63
  end
60
64
 
@@ -84,21 +88,35 @@ module Backup
84
88
  email.cc = cc
85
89
  email.bcc = bcc
86
90
  email.reply_to = reply_to
87
- email.subject = message.call(model, :status => status_data_for(status))
91
+ email.subject = message.call(model, status: status_data_for(status))
92
+
93
+ # By default, the `mail` gem doesn't include BCC in raw output, which is
94
+ # needed for SES to send to those addresses.
95
+ email[:bcc].include_in_headers = true
88
96
 
89
97
  send_log = send_log_on.include?(status)
90
- template = Backup::Template.new({ :model => model, :send_log => send_log })
91
- email.body = template.result('notifier/mail/%s.erb' % status.to_s)
98
+ template = Backup::Template.new(model: model, send_log: send_log)
99
+ email.body = template.result(sprintf("notifier/mail/%s.erb", status.to_s))
92
100
 
93
101
  if send_log
94
102
  email.convert_to_multipart
95
- email.attachments["#{ model.time }.#{ model.trigger }.log"] = {
96
- :mime_type => 'text/plain;',
97
- :content => Logger.messages.map(&:formatted_lines).flatten.join("\n")
103
+ email.attachments["#{model.time}.#{model.trigger}.log"] = {
104
+ mime_type: "text/plain;",
105
+ content: Logger.messages.map(&:formatted_lines).flatten.join("\n")
106
+ }
107
+ end
108
+
109
+ send_opts = {
110
+ raw_message: {
111
+ data: email.to_s
98
112
  }
113
+ }
114
+
115
+ if email.respond_to?(:destinations)
116
+ send_opts[:destinations] = email.destinations
99
117
  end
100
118
 
101
- client.send_raw_email(email)
119
+ client.send_raw_email(send_opts)
102
120
  end
103
121
  end
104
122
  end
@@ -1,11 +1,9 @@
1
- # encoding: utf-8
2
- require 'uri'
3
- require 'json'
1
+ require "uri"
2
+ require "json"
4
3
 
5
4
  module Backup
6
5
  module Notifier
7
6
  class Slack < Base
8
-
9
7
  ##
10
8
  # The incoming webhook url
11
9
  attr_accessor :webhook_url
@@ -38,7 +36,7 @@ module Backup
38
36
  instance_eval(&block) if block_given?
39
37
 
40
38
  @send_log_on ||= [:warning, :failure]
41
- @icon_emoji ||= ':floppy_disk:'
39
+ @icon_emoji ||= ":floppy_disk:"
42
40
  end
43
41
 
44
42
  private
@@ -62,8 +60,8 @@ module Backup
62
60
  #
63
61
  def notify!(status)
64
62
  data = {
65
- :text => message.call(model, :status => status_data_for(status)),
66
- :attachments => [attachment(status)]
63
+ text: message.call(model, status: status_data_for(status)),
64
+ attachments: [attachment(status)]
67
65
  }
68
66
  [:channel, :username, :icon_emoji].each do |param|
69
67
  val = send(param)
@@ -71,43 +69,43 @@ module Backup
71
69
  end
72
70
 
73
71
  options = {
74
- :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
75
- :body => URI.encode_www_form(:payload => JSON.dump(data))
72
+ headers: { "Content-Type" => "application/x-www-form-urlencoded" },
73
+ body: URI.encode_www_form(payload: JSON.dump(data))
76
74
  }
77
- options.merge!(:expects => 200) # raise error if unsuccessful
75
+ options[:expects] = 200 # raise error if unsuccessful
78
76
  Excon.post(uri, options)
79
77
  end
80
78
 
81
79
  def attachment(status)
82
80
  {
83
- :fallback => "#{title(status)} - Job: #{model.label} (#{model.trigger})",
84
- :text => title(status),
85
- :color => color(status),
86
- :fields => [
81
+ fallback: "#{title(status)} - Job: #{model.label} (#{model.trigger})",
82
+ text: title(status),
83
+ color: color(status),
84
+ fields: [
87
85
  {
88
- :title => "Job",
89
- :value => "#{model.label} (#{model.trigger})",
90
- :short => false
86
+ title: "Job",
87
+ value: "#{model.label} (#{model.trigger})",
88
+ short: false
91
89
  },
92
90
  {
93
- :title => "Started",
94
- :value => model.started_at,
95
- :short => true
91
+ title: "Started",
92
+ value: model.started_at,
93
+ short: true
96
94
  },
97
95
  {
98
- :title => "Finished",
99
- :value => model.finished_at,
100
- :short => true
96
+ title: "Finished",
97
+ value: model.finished_at,
98
+ short: true
101
99
  },
102
100
  {
103
- :title => "Duration",
104
- :value => model.duration,
105
- :short => true
101
+ title: "Duration",
102
+ value: model.duration,
103
+ short: true
106
104
  },
107
105
  {
108
- :title => "Version",
109
- :value => "Backup v#{Backup::VERSION}\nRuby: #{RUBY_DESCRIPTION}",
110
- :short => false
106
+ title: "Version",
107
+ value: "Backup v#{Backup::VERSION}\nRuby: #{RUBY_DESCRIPTION}",
108
+ short: false
111
109
  },
112
110
  log_field(status)
113
111
  ].compact
@@ -116,27 +114,28 @@ module Backup
116
114
 
117
115
  def log_field(status)
118
116
  send_log = send_log_on.include?(status)
117
+ return unless send_log
119
118
 
120
- return {
121
- :title => "Detailed Backup Log",
122
- :value => Logger.messages.map(&:formatted_lines).flatten.join("\n"),
123
- :short => false,
124
- } if send_log
119
+ {
120
+ title: "Detailed Backup Log",
121
+ value: Logger.messages.map(&:formatted_lines).flatten.join("\n"),
122
+ short: false
123
+ }
125
124
  end
126
125
 
127
126
  def color(status)
128
127
  case status
129
- when :success then 'good'
130
- when :failure then 'danger'
131
- when :warning then 'warning'
128
+ when :success then "good"
129
+ when :failure then "danger"
130
+ when :warning then "warning"
132
131
  end
133
132
  end
134
133
 
135
134
  def title(status)
136
135
  case status
137
- when :success then 'Backup Completed Successfully!'
138
- when :failure then 'Backup Failed!'
139
- when :warning then 'Backup Completed Successfully (with Warnings)!'
136
+ when :success then "Backup Completed Successfully!"
137
+ when :failure then "Backup Failed!"
138
+ when :warning then "Backup Completed Successfully (with Warnings)!"
140
139
  end
141
140
  end
142
141
 
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'twitter'
1
+ require "twitter"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Twitter < Base
7
-
8
6
  ##
9
7
  # Twitter consumer key credentials
10
8
  attr_accessor :consumer_key, :consumer_secret
@@ -38,7 +36,7 @@ module Backup
38
36
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
39
37
  #
40
38
  def notify!(status)
41
- send_message(message.call(model, :status => status_data_for(status)))
39
+ send_message(message.call(model, status: status_data_for(status)))
42
40
  end
43
41
 
44
42
  # Twitter::Client will raise an error if unsuccessful.
@@ -52,7 +50,6 @@ module Backup
52
50
 
53
51
  client.update(message)
54
52
  end
55
-
56
53
  end
57
54
  end
58
55
  end
@@ -1,9 +1,6 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Notifier
5
3
  class Zabbix < Base
6
-
7
4
  attr_accessor :zabbix_host
8
5
 
9
6
  attr_accessor :zabbix_port
@@ -19,10 +16,10 @@ module Backup
19
16
  instance_eval(&block) if block_given?
20
17
 
21
18
  @zabbix_host ||= Config.hostname
22
- @zabbix_port ||= 10051
23
- @service_name ||= "Backup #{ model.trigger }"
19
+ @zabbix_port ||= 10_051
20
+ @service_name ||= "Backup #{model.trigger}"
24
21
  @service_host ||= Config.hostname
25
- @item_key ||= 'backup_status'
22
+ @item_key ||= "backup_status"
26
23
  end
27
24
 
28
25
  private
@@ -45,18 +42,18 @@ module Backup
45
42
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
46
43
  #
47
44
  def notify!(status)
48
- send_message(message.call(model, :status => status_data_for(status)))
45
+ send_message(message.call(model, status: status_data_for(status)))
49
46
  end
50
47
 
51
48
  def send_message(message)
52
49
  msg = [service_host, service_name, model.exit_status, message].join("\t")
53
- cmd = "#{ utility(:zabbix_sender) }" +
54
- " -z '#{ zabbix_host }'" +
55
- " -p '#{ zabbix_port }'" +
56
- " -s #{ service_host }" +
57
- " -k #{ item_key }" +
58
- " -o '#{ msg }'"
59
- run("echo '#{ msg }' | #{ cmd }")
50
+ cmd = utility(:zabbix_sender).to_s +
51
+ " -z '#{zabbix_host}'" \
52
+ " -p '#{zabbix_port}'" \
53
+ " -s #{service_host}" \
54
+ " -k #{item_key}" \
55
+ " -o '#{msg}'"
56
+ run("echo '#{msg}' | #{cmd}")
60
57
  end
61
58
  end
62
59
  end