backup 4.4.1 → 5.0.0.beta.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (80) hide show
  1. checksums.yaml +5 -5
  2. data/LICENSE +19 -0
  3. data/README.md +1 -1
  4. data/lib/backup.rb +74 -78
  5. data/lib/backup/archive.rb +31 -32
  6. data/lib/backup/binder.rb +2 -6
  7. data/lib/backup/cleaner.rb +14 -18
  8. data/lib/backup/cli.rb +104 -108
  9. data/lib/backup/cloud_io/base.rb +4 -7
  10. data/lib/backup/cloud_io/cloud_files.rb +60 -62
  11. data/lib/backup/cloud_io/s3.rb +69 -76
  12. data/lib/backup/compressor/base.rb +4 -7
  13. data/lib/backup/compressor/bzip2.rb +3 -7
  14. data/lib/backup/compressor/custom.rb +2 -6
  15. data/lib/backup/compressor/gzip.rb +16 -17
  16. data/lib/backup/config.rb +17 -18
  17. data/lib/backup/config/dsl.rb +16 -17
  18. data/lib/backup/config/helpers.rb +10 -16
  19. data/lib/backup/database/base.rb +22 -21
  20. data/lib/backup/database/mongodb.rb +36 -37
  21. data/lib/backup/database/mysql.rb +40 -41
  22. data/lib/backup/database/openldap.rb +8 -10
  23. data/lib/backup/database/postgresql.rb +29 -30
  24. data/lib/backup/database/redis.rb +27 -30
  25. data/lib/backup/database/riak.rb +15 -18
  26. data/lib/backup/database/sqlite.rb +4 -6
  27. data/lib/backup/encryptor/base.rb +2 -4
  28. data/lib/backup/encryptor/gpg.rb +49 -59
  29. data/lib/backup/encryptor/open_ssl.rb +11 -14
  30. data/lib/backup/errors.rb +7 -12
  31. data/lib/backup/logger.rb +16 -18
  32. data/lib/backup/logger/console.rb +5 -8
  33. data/lib/backup/logger/fog_adapter.rb +2 -6
  34. data/lib/backup/logger/logfile.rb +10 -12
  35. data/lib/backup/logger/syslog.rb +2 -4
  36. data/lib/backup/model.rb +75 -40
  37. data/lib/backup/notifier/base.rb +24 -26
  38. data/lib/backup/notifier/campfire.rb +9 -11
  39. data/lib/backup/notifier/command.rb +0 -3
  40. data/lib/backup/notifier/datadog.rb +9 -12
  41. data/lib/backup/notifier/flowdock.rb +13 -17
  42. data/lib/backup/notifier/hipchat.rb +11 -13
  43. data/lib/backup/notifier/http_post.rb +11 -14
  44. data/lib/backup/notifier/mail.rb +44 -47
  45. data/lib/backup/notifier/nagios.rb +5 -9
  46. data/lib/backup/notifier/pagerduty.rb +10 -12
  47. data/lib/backup/notifier/prowl.rb +15 -15
  48. data/lib/backup/notifier/pushover.rb +7 -10
  49. data/lib/backup/notifier/ses.rb +34 -16
  50. data/lib/backup/notifier/slack.rb +39 -40
  51. data/lib/backup/notifier/twitter.rb +2 -5
  52. data/lib/backup/notifier/zabbix.rb +11 -14
  53. data/lib/backup/package.rb +5 -9
  54. data/lib/backup/packager.rb +16 -17
  55. data/lib/backup/pipeline.rb +17 -21
  56. data/lib/backup/splitter.rb +8 -11
  57. data/lib/backup/storage/base.rb +5 -8
  58. data/lib/backup/storage/cloud_files.rb +21 -23
  59. data/lib/backup/storage/cycler.rb +10 -15
  60. data/lib/backup/storage/dropbox.rb +15 -21
  61. data/lib/backup/storage/ftp.rb +8 -10
  62. data/lib/backup/storage/local.rb +5 -8
  63. data/lib/backup/storage/qiniu.rb +8 -8
  64. data/lib/backup/storage/rsync.rb +24 -26
  65. data/lib/backup/storage/s3.rb +27 -28
  66. data/lib/backup/storage/scp.rb +10 -12
  67. data/lib/backup/storage/sftp.rb +10 -12
  68. data/lib/backup/syncer/base.rb +5 -8
  69. data/lib/backup/syncer/cloud/base.rb +27 -30
  70. data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
  71. data/lib/backup/syncer/cloud/local_file.rb +5 -8
  72. data/lib/backup/syncer/cloud/s3.rb +23 -24
  73. data/lib/backup/syncer/rsync/base.rb +6 -10
  74. data/lib/backup/syncer/rsync/local.rb +1 -5
  75. data/lib/backup/syncer/rsync/pull.rb +6 -10
  76. data/lib/backup/syncer/rsync/push.rb +18 -22
  77. data/lib/backup/template.rb +9 -14
  78. data/lib/backup/utilities.rb +82 -69
  79. data/lib/backup/version.rb +1 -3
  80. metadata +100 -660
@@ -1,9 +1,6 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Notifier
5
3
  class Nagios < Base
6
-
7
4
  ##
8
5
  # Host of Nagios server to notify on backup completion.
9
6
  attr_accessor :nagios_host
@@ -30,8 +27,8 @@ module Backup
30
27
 
31
28
  @nagios_host ||= Config.hostname
32
29
  @nagios_port ||= 5667
33
- @send_nsca_cfg||= "/etc/nagios/send_nsca.cfg"
34
- @service_name ||= "Backup #{ model.trigger }"
30
+ @send_nsca_cfg ||= "/etc/nagios/send_nsca.cfg"
31
+ @service_name ||= "Backup #{model.trigger}"
35
32
  @service_host ||= Config.hostname
36
33
  end
37
34
 
@@ -55,15 +52,14 @@ module Backup
55
52
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
56
53
  #
57
54
  def notify!(status)
58
- send_message(message.call(model, :status => status_data_for(status)))
55
+ send_message(message.call(model, status: status_data_for(status)))
59
56
  end
60
57
 
61
58
  def send_message(message)
62
- cmd = "#{ utility(:send_nsca) } -H '#{ nagios_host }' -p '#{ nagios_port }' -c '#{ send_nsca_cfg }'"
59
+ cmd = "#{utility(:send_nsca)} -H '#{nagios_host}' -p '#{nagios_port}' -c '#{send_nsca_cfg}'"
63
60
  msg = [service_host, service_name, model.exit_status, message].join("\t")
64
- run("echo '#{ msg }' | #{ cmd }")
61
+ run("echo '#{msg}' | #{cmd}")
65
62
  end
66
-
67
63
  end
68
64
  end
69
65
  end
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'pagerduty'
1
+ require "pagerduty"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class PagerDuty < Base
7
-
8
6
  ##
9
7
  # PagerDuty Service API Key. Should be a 32 character hex string.
10
8
  attr_accessor :service_key
@@ -46,15 +44,15 @@ module Backup
46
44
  incident_description = "Backup - #{model.label}"
47
45
  incident_key = "backup/#{model.trigger}"
48
46
  incident_details = {
49
- :incident_key => incident_key,
50
- :details => {
51
- :trigger => model.trigger,
52
- :label => model.label,
53
- :started_at => model.started_at,
54
- :finished_at => model.finished_at,
55
- :duration => model.duration,
56
- :status => status,
57
- :exception => model.exception
47
+ incident_key: incident_key,
48
+ details: {
49
+ trigger: model.trigger,
50
+ label: model.label,
51
+ started_at: model.started_at,
52
+ finished_at: model.finished_at,
53
+ duration: model.duration,
54
+ status: status,
55
+ exception: model.exception
58
56
  }
59
57
  }
60
58
 
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'uri'
1
+ require "uri"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Prowl < Base
7
-
8
6
  ##
9
7
  # Application name
10
8
  # Tell something like your server name. Example: "Server1 Backup"
@@ -16,10 +14,13 @@ module Backup
16
14
  attr_accessor :api_key
17
15
 
18
16
  def initialize(model, &block)
19
- @message = lambda do |model, data|
20
- "#{ model.label } (#{ model.trigger })"
21
- end
17
+ @message =
18
+ lambda do |m, _|
19
+ "#{m.label} (#{m.trigger})"
20
+ end
21
+
22
22
  super
23
+
23
24
  instance_eval(&block) if block_given?
24
25
  end
25
26
 
@@ -47,22 +48,21 @@ module Backup
47
48
  end
48
49
 
49
50
  def send_message(status)
50
- uri = 'https://api.prowlapp.com/publicapi/add'
51
+ uri = "https://api.prowlapp.com/publicapi/add"
51
52
  status_data = status_data_for(status)
52
53
  data = {
53
- :application => application,
54
- :apikey => api_key,
55
- :event => status_data[:message],
56
- :description => message.call(model, :status => status_data)
54
+ application: application,
55
+ apikey: api_key,
56
+ event: status_data[:message],
57
+ description: message.call(model, status: status_data)
57
58
  }
58
59
  options = {
59
- :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
60
- :body => URI.encode_www_form(data)
60
+ headers: { "Content-Type" => "application/x-www-form-urlencoded" },
61
+ body: URI.encode_www_form(data)
61
62
  }
62
- options.merge!(:expects => 200) # raise error if unsuccessful
63
+ options[:expects] = 200 # raise error if unsuccessful
63
64
  Excon.post(uri, options)
64
65
  end
65
-
66
66
  end
67
67
  end
68
68
  end
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'uri'
1
+ require "uri"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Pushover < Base
7
-
8
6
  ##
9
7
  # The API User Token
10
8
  attr_accessor :user
@@ -51,24 +49,23 @@ module Backup
51
49
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
52
50
  #
53
51
  def notify!(status)
54
- send_message(message.call(model, :status => status_data_for(status)))
52
+ send_message(message.call(model, status: status_data_for(status)))
55
53
  end
56
54
 
57
55
  def send_message(message)
58
- uri = 'https://api.pushover.net/1/messages.json'
59
- data = { :user => user, :token => token, :message => message }
56
+ uri = "https://api.pushover.net/1/messages.json"
57
+ data = { user: user, token: token, message: message }
60
58
  [:device, :title, :priority].each do |param|
61
59
  val = send(param)
62
60
  data.merge!(param => val) if val
63
61
  end
64
62
  options = {
65
- :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
66
- :body => URI.encode_www_form(data)
63
+ headers: { "Content-Type" => "application/x-www-form-urlencoded" },
64
+ body: URI.encode_www_form(data)
67
65
  }
68
- options.merge!(:expects => 200) # raise error if unsuccessful
66
+ options[:expects] = 200 # raise error if unsuccessful
69
67
  Excon.post(uri, options)
70
68
  end
71
-
72
69
  end
73
70
  end
74
71
  end
@@ -1,13 +1,12 @@
1
- # encoding: utf-8
2
- require 'aws/ses'
1
+ require "aws-sdk"
2
+ require "mail"
3
3
 
4
4
  module Backup
5
5
  module Notifier
6
6
  class Ses < Base
7
-
8
7
  ##
9
8
  # Amazon Simple Email Service (SES) Credentials
10
- attr_accessor :access_key_id, :secret_access_key
9
+ attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
11
10
 
12
11
  ##
13
12
  # SES Region
@@ -37,7 +36,7 @@ module Backup
37
36
  super
38
37
  instance_eval(&block) if block_given?
39
38
 
40
- @region ||= 'eu-west-1'
39
+ @region ||= "eu-west-1"
41
40
  @send_log_on ||= [:warning, :failure]
42
41
  end
43
42
 
@@ -51,10 +50,15 @@ module Backup
51
50
  private
52
51
 
53
52
  def client
54
- AWS::SES::Base.new(
55
- :access_key_id => access_key_id,
56
- :secret_access_key => secret_access_key,
57
- :server => "email.#{region}.amazonaws.com"
53
+ credentials = if use_iam_profile
54
+ Aws::InstanceProfileCredentials.new
55
+ else
56
+ Aws::Credentials.new(access_key_id, secret_access_key)
57
+ end
58
+
59
+ Aws::SES::Client.new(
60
+ region: region,
61
+ credentials: credentials
58
62
  )
59
63
  end
60
64
 
@@ -84,21 +88,35 @@ module Backup
84
88
  email.cc = cc
85
89
  email.bcc = bcc
86
90
  email.reply_to = reply_to
87
- email.subject = message.call(model, :status => status_data_for(status))
91
+ email.subject = message.call(model, status: status_data_for(status))
92
+
93
+ # By default, the `mail` gem doesn't include BCC in raw output, which is
94
+ # needed for SES to send to those addresses.
95
+ email[:bcc].include_in_headers = true
88
96
 
89
97
  send_log = send_log_on.include?(status)
90
- template = Backup::Template.new({ :model => model, :send_log => send_log })
91
- email.body = template.result('notifier/mail/%s.erb' % status.to_s)
98
+ template = Backup::Template.new(model: model, send_log: send_log)
99
+ email.body = template.result(sprintf("notifier/mail/%s.erb", status.to_s))
92
100
 
93
101
  if send_log
94
102
  email.convert_to_multipart
95
- email.attachments["#{ model.time }.#{ model.trigger }.log"] = {
96
- :mime_type => 'text/plain;',
97
- :content => Logger.messages.map(&:formatted_lines).flatten.join("\n")
103
+ email.attachments["#{model.time}.#{model.trigger}.log"] = {
104
+ mime_type: "text/plain;",
105
+ content: Logger.messages.map(&:formatted_lines).flatten.join("\n")
106
+ }
107
+ end
108
+
109
+ send_opts = {
110
+ raw_message: {
111
+ data: email.to_s
98
112
  }
113
+ }
114
+
115
+ if email.respond_to?(:destinations)
116
+ send_opts[:destinations] = email.destinations
99
117
  end
100
118
 
101
- client.send_raw_email(email)
119
+ client.send_raw_email(send_opts)
102
120
  end
103
121
  end
104
122
  end
@@ -1,11 +1,9 @@
1
- # encoding: utf-8
2
- require 'uri'
3
- require 'json'
1
+ require "uri"
2
+ require "json"
4
3
 
5
4
  module Backup
6
5
  module Notifier
7
6
  class Slack < Base
8
-
9
7
  ##
10
8
  # The incoming webhook url
11
9
  attr_accessor :webhook_url
@@ -38,7 +36,7 @@ module Backup
38
36
  instance_eval(&block) if block_given?
39
37
 
40
38
  @send_log_on ||= [:warning, :failure]
41
- @icon_emoji ||= ':floppy_disk:'
39
+ @icon_emoji ||= ":floppy_disk:"
42
40
  end
43
41
 
44
42
  private
@@ -62,8 +60,8 @@ module Backup
62
60
  #
63
61
  def notify!(status)
64
62
  data = {
65
- :text => message.call(model, :status => status_data_for(status)),
66
- :attachments => [attachment(status)]
63
+ text: message.call(model, status: status_data_for(status)),
64
+ attachments: [attachment(status)]
67
65
  }
68
66
  [:channel, :username, :icon_emoji].each do |param|
69
67
  val = send(param)
@@ -71,43 +69,43 @@ module Backup
71
69
  end
72
70
 
73
71
  options = {
74
- :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
75
- :body => URI.encode_www_form(:payload => JSON.dump(data))
72
+ headers: { "Content-Type" => "application/x-www-form-urlencoded" },
73
+ body: URI.encode_www_form(payload: JSON.dump(data))
76
74
  }
77
- options.merge!(:expects => 200) # raise error if unsuccessful
75
+ options[:expects] = 200 # raise error if unsuccessful
78
76
  Excon.post(uri, options)
79
77
  end
80
78
 
81
79
  def attachment(status)
82
80
  {
83
- :fallback => "#{title(status)} - Job: #{model.label} (#{model.trigger})",
84
- :text => title(status),
85
- :color => color(status),
86
- :fields => [
81
+ fallback: "#{title(status)} - Job: #{model.label} (#{model.trigger})",
82
+ text: title(status),
83
+ color: color(status),
84
+ fields: [
87
85
  {
88
- :title => "Job",
89
- :value => "#{model.label} (#{model.trigger})",
90
- :short => false
86
+ title: "Job",
87
+ value: "#{model.label} (#{model.trigger})",
88
+ short: false
91
89
  },
92
90
  {
93
- :title => "Started",
94
- :value => model.started_at,
95
- :short => true
91
+ title: "Started",
92
+ value: model.started_at,
93
+ short: true
96
94
  },
97
95
  {
98
- :title => "Finished",
99
- :value => model.finished_at,
100
- :short => true
96
+ title: "Finished",
97
+ value: model.finished_at,
98
+ short: true
101
99
  },
102
100
  {
103
- :title => "Duration",
104
- :value => model.duration,
105
- :short => true
101
+ title: "Duration",
102
+ value: model.duration,
103
+ short: true
106
104
  },
107
105
  {
108
- :title => "Version",
109
- :value => "Backup v#{Backup::VERSION}\nRuby: #{RUBY_DESCRIPTION}",
110
- :short => false
106
+ title: "Version",
107
+ value: "Backup v#{Backup::VERSION}\nRuby: #{RUBY_DESCRIPTION}",
108
+ short: false
111
109
  },
112
110
  log_field(status)
113
111
  ].compact
@@ -116,27 +114,28 @@ module Backup
116
114
 
117
115
  def log_field(status)
118
116
  send_log = send_log_on.include?(status)
117
+ return unless send_log
119
118
 
120
- return {
121
- :title => "Detailed Backup Log",
122
- :value => Logger.messages.map(&:formatted_lines).flatten.join("\n"),
123
- :short => false,
124
- } if send_log
119
+ {
120
+ title: "Detailed Backup Log",
121
+ value: Logger.messages.map(&:formatted_lines).flatten.join("\n"),
122
+ short: false
123
+ }
125
124
  end
126
125
 
127
126
  def color(status)
128
127
  case status
129
- when :success then 'good'
130
- when :failure then 'danger'
131
- when :warning then 'warning'
128
+ when :success then "good"
129
+ when :failure then "danger"
130
+ when :warning then "warning"
132
131
  end
133
132
  end
134
133
 
135
134
  def title(status)
136
135
  case status
137
- when :success then 'Backup Completed Successfully!'
138
- when :failure then 'Backup Failed!'
139
- when :warning then 'Backup Completed Successfully (with Warnings)!'
136
+ when :success then "Backup Completed Successfully!"
137
+ when :failure then "Backup Failed!"
138
+ when :warning then "Backup Completed Successfully (with Warnings)!"
140
139
  end
141
140
  end
142
141
 
@@ -1,10 +1,8 @@
1
- # encoding: utf-8
2
- require 'twitter'
1
+ require "twitter"
3
2
 
4
3
  module Backup
5
4
  module Notifier
6
5
  class Twitter < Base
7
-
8
6
  ##
9
7
  # Twitter consumer key credentials
10
8
  attr_accessor :consumer_key, :consumer_secret
@@ -38,7 +36,7 @@ module Backup
38
36
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
39
37
  #
40
38
  def notify!(status)
41
- send_message(message.call(model, :status => status_data_for(status)))
39
+ send_message(message.call(model, status: status_data_for(status)))
42
40
  end
43
41
 
44
42
  # Twitter::Client will raise an error if unsuccessful.
@@ -52,7 +50,6 @@ module Backup
52
50
 
53
51
  client.update(message)
54
52
  end
55
-
56
53
  end
57
54
  end
58
55
  end
@@ -1,9 +1,6 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Notifier
5
3
  class Zabbix < Base
6
-
7
4
  attr_accessor :zabbix_host
8
5
 
9
6
  attr_accessor :zabbix_port
@@ -19,10 +16,10 @@ module Backup
19
16
  instance_eval(&block) if block_given?
20
17
 
21
18
  @zabbix_host ||= Config.hostname
22
- @zabbix_port ||= 10051
23
- @service_name ||= "Backup #{ model.trigger }"
19
+ @zabbix_port ||= 10_051
20
+ @service_name ||= "Backup #{model.trigger}"
24
21
  @service_host ||= Config.hostname
25
- @item_key ||= 'backup_status'
22
+ @item_key ||= "backup_status"
26
23
  end
27
24
 
28
25
  private
@@ -45,18 +42,18 @@ module Backup
45
42
  # : Notification will be sent if `on_warning` or `on_success` is `true`.
46
43
  #
47
44
  def notify!(status)
48
- send_message(message.call(model, :status => status_data_for(status)))
45
+ send_message(message.call(model, status: status_data_for(status)))
49
46
  end
50
47
 
51
48
  def send_message(message)
52
49
  msg = [service_host, service_name, model.exit_status, message].join("\t")
53
- cmd = "#{ utility(:zabbix_sender) }" +
54
- " -z '#{ zabbix_host }'" +
55
- " -p '#{ zabbix_port }'" +
56
- " -s #{ service_host }" +
57
- " -k #{ item_key }" +
58
- " -o '#{ msg }'"
59
- run("echo '#{ msg }' | #{ cmd }")
50
+ cmd = utility(:zabbix_sender).to_s +
51
+ " -z '#{zabbix_host}'" \
52
+ " -p '#{zabbix_port}'" \
53
+ " -s #{service_host}" \
54
+ " -k #{item_key}" \
55
+ " -o '#{msg}'"
56
+ run("echo '#{msg}' | #{cmd}")
60
57
  end
61
58
  end
62
59
  end