backup-bouchard 4.4.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (134) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +19 -0
  3. data/README.md +29 -0
  4. data/bin/backup +5 -0
  5. data/lib/backup.rb +140 -0
  6. data/lib/backup/archive.rb +169 -0
  7. data/lib/backup/binder.rb +18 -0
  8. data/lib/backup/cleaner.rb +112 -0
  9. data/lib/backup/cli.rb +370 -0
  10. data/lib/backup/cloud_io/base.rb +38 -0
  11. data/lib/backup/cloud_io/cloud_files.rb +296 -0
  12. data/lib/backup/cloud_io/s3.rb +253 -0
  13. data/lib/backup/compressor/base.rb +32 -0
  14. data/lib/backup/compressor/bzip2.rb +35 -0
  15. data/lib/backup/compressor/custom.rb +49 -0
  16. data/lib/backup/compressor/gzip.rb +73 -0
  17. data/lib/backup/config.rb +118 -0
  18. data/lib/backup/config/dsl.rb +100 -0
  19. data/lib/backup/config/helpers.rb +137 -0
  20. data/lib/backup/database/base.rb +86 -0
  21. data/lib/backup/database/mongodb.rb +187 -0
  22. data/lib/backup/database/mysql.rb +191 -0
  23. data/lib/backup/database/openldap.rb +93 -0
  24. data/lib/backup/database/postgresql.rb +132 -0
  25. data/lib/backup/database/redis.rb +177 -0
  26. data/lib/backup/database/riak.rb +79 -0
  27. data/lib/backup/database/sqlite.rb +55 -0
  28. data/lib/backup/encryptor/base.rb +27 -0
  29. data/lib/backup/encryptor/gpg.rb +740 -0
  30. data/lib/backup/encryptor/open_ssl.rb +74 -0
  31. data/lib/backup/errors.rb +53 -0
  32. data/lib/backup/logger.rb +197 -0
  33. data/lib/backup/logger/console.rb +48 -0
  34. data/lib/backup/logger/fog_adapter.rb +25 -0
  35. data/lib/backup/logger/logfile.rb +131 -0
  36. data/lib/backup/logger/syslog.rb +114 -0
  37. data/lib/backup/model.rb +477 -0
  38. data/lib/backup/notifier/base.rb +126 -0
  39. data/lib/backup/notifier/campfire.rb +61 -0
  40. data/lib/backup/notifier/command.rb +99 -0
  41. data/lib/backup/notifier/datadog.rb +104 -0
  42. data/lib/backup/notifier/flowdock.rb +99 -0
  43. data/lib/backup/notifier/hipchat.rb +116 -0
  44. data/lib/backup/notifier/http_post.rb +114 -0
  45. data/lib/backup/notifier/mail.rb +246 -0
  46. data/lib/backup/notifier/nagios.rb +65 -0
  47. data/lib/backup/notifier/pagerduty.rb +79 -0
  48. data/lib/backup/notifier/prowl.rb +68 -0
  49. data/lib/backup/notifier/pushover.rb +71 -0
  50. data/lib/backup/notifier/ses.rb +103 -0
  51. data/lib/backup/notifier/slack.rb +147 -0
  52. data/lib/backup/notifier/twitter.rb +55 -0
  53. data/lib/backup/notifier/zabbix.rb +60 -0
  54. data/lib/backup/package.rb +51 -0
  55. data/lib/backup/packager.rb +105 -0
  56. data/lib/backup/pipeline.rb +120 -0
  57. data/lib/backup/splitter.rb +73 -0
  58. data/lib/backup/storage/base.rb +66 -0
  59. data/lib/backup/storage/cloud_files.rb +156 -0
  60. data/lib/backup/storage/cycler.rb +70 -0
  61. data/lib/backup/storage/dropbox.rb +210 -0
  62. data/lib/backup/storage/ftp.rb +110 -0
  63. data/lib/backup/storage/local.rb +61 -0
  64. data/lib/backup/storage/qiniu.rb +65 -0
  65. data/lib/backup/storage/rsync.rb +246 -0
  66. data/lib/backup/storage/s3.rb +155 -0
  67. data/lib/backup/storage/scp.rb +65 -0
  68. data/lib/backup/storage/sftp.rb +80 -0
  69. data/lib/backup/syncer/base.rb +67 -0
  70. data/lib/backup/syncer/cloud/base.rb +176 -0
  71. data/lib/backup/syncer/cloud/cloud_files.rb +81 -0
  72. data/lib/backup/syncer/cloud/local_file.rb +97 -0
  73. data/lib/backup/syncer/cloud/s3.rb +109 -0
  74. data/lib/backup/syncer/rsync/base.rb +50 -0
  75. data/lib/backup/syncer/rsync/local.rb +27 -0
  76. data/lib/backup/syncer/rsync/pull.rb +47 -0
  77. data/lib/backup/syncer/rsync/push.rb +201 -0
  78. data/lib/backup/template.rb +41 -0
  79. data/lib/backup/utilities.rb +228 -0
  80. data/lib/backup/version.rb +3 -0
  81. data/templates/cli/archive +28 -0
  82. data/templates/cli/compressor/bzip2 +4 -0
  83. data/templates/cli/compressor/custom +7 -0
  84. data/templates/cli/compressor/gzip +4 -0
  85. data/templates/cli/config +123 -0
  86. data/templates/cli/databases/mongodb +15 -0
  87. data/templates/cli/databases/mysql +18 -0
  88. data/templates/cli/databases/openldap +24 -0
  89. data/templates/cli/databases/postgresql +16 -0
  90. data/templates/cli/databases/redis +16 -0
  91. data/templates/cli/databases/riak +17 -0
  92. data/templates/cli/databases/sqlite +11 -0
  93. data/templates/cli/encryptor/gpg +27 -0
  94. data/templates/cli/encryptor/openssl +9 -0
  95. data/templates/cli/model +26 -0
  96. data/templates/cli/notifier/zabbix +15 -0
  97. data/templates/cli/notifiers/campfire +12 -0
  98. data/templates/cli/notifiers/command +32 -0
  99. data/templates/cli/notifiers/datadog +57 -0
  100. data/templates/cli/notifiers/flowdock +16 -0
  101. data/templates/cli/notifiers/hipchat +16 -0
  102. data/templates/cli/notifiers/http_post +32 -0
  103. data/templates/cli/notifiers/mail +24 -0
  104. data/templates/cli/notifiers/nagios +13 -0
  105. data/templates/cli/notifiers/pagerduty +12 -0
  106. data/templates/cli/notifiers/prowl +11 -0
  107. data/templates/cli/notifiers/pushover +11 -0
  108. data/templates/cli/notifiers/ses +15 -0
  109. data/templates/cli/notifiers/slack +22 -0
  110. data/templates/cli/notifiers/twitter +13 -0
  111. data/templates/cli/splitter +7 -0
  112. data/templates/cli/storages/cloud_files +11 -0
  113. data/templates/cli/storages/dropbox +20 -0
  114. data/templates/cli/storages/ftp +13 -0
  115. data/templates/cli/storages/local +8 -0
  116. data/templates/cli/storages/qiniu +12 -0
  117. data/templates/cli/storages/rsync +17 -0
  118. data/templates/cli/storages/s3 +16 -0
  119. data/templates/cli/storages/scp +15 -0
  120. data/templates/cli/storages/sftp +15 -0
  121. data/templates/cli/syncers/cloud_files +22 -0
  122. data/templates/cli/syncers/rsync_local +20 -0
  123. data/templates/cli/syncers/rsync_pull +28 -0
  124. data/templates/cli/syncers/rsync_push +28 -0
  125. data/templates/cli/syncers/s3 +27 -0
  126. data/templates/general/links +3 -0
  127. data/templates/general/version.erb +2 -0
  128. data/templates/notifier/mail/failure.erb +16 -0
  129. data/templates/notifier/mail/success.erb +16 -0
  130. data/templates/notifier/mail/warning.erb +16 -0
  131. data/templates/storage/dropbox/authorization_url.erb +6 -0
  132. data/templates/storage/dropbox/authorized.erb +4 -0
  133. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  134. metadata +518 -0
@@ -0,0 +1,116 @@
1
+ require "hipchat"
2
+
3
+ module Backup
4
+ module Notifier
5
+ class Hipchat < Base
6
+ ##
7
+ # The Hipchat API token
8
+ attr_accessor :token
9
+
10
+ ##
11
+ # The Hipchat API version
12
+ # Either 'v1' or 'v2' (default is 'v1')
13
+ attr_accessor :api_version
14
+
15
+ ##
16
+ # Who the notification should appear from
17
+ attr_accessor :from
18
+
19
+ ##
20
+ # Custom server URL
21
+ attr_accessor :server_url
22
+
23
+ ##
24
+ # The rooms that should be notified
25
+ attr_accessor :rooms_notified
26
+
27
+ ##
28
+ # Notify users in the room
29
+ attr_accessor :notify_users
30
+
31
+ ##
32
+ # The background color of a success message.
33
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
34
+ attr_accessor :success_color
35
+
36
+ ##
37
+ # The background color of a warning message.
38
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
39
+ attr_accessor :warning_color
40
+
41
+ ##
42
+ # The background color of an error message.
43
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
44
+ attr_accessor :failure_color
45
+
46
+ def initialize(model, &block)
47
+ super
48
+ instance_eval(&block) if block_given?
49
+
50
+ @notify_users ||= false
51
+ @rooms_notified ||= []
52
+ @success_color ||= "yellow"
53
+ @warning_color ||= "yellow"
54
+ @failure_color ||= "yellow"
55
+ @api_version ||= "v1"
56
+ end
57
+
58
+ private
59
+
60
+ ##
61
+ # Notify the user of the backup operation results.
62
+ #
63
+ # `status` indicates one of the following:
64
+ #
65
+ # `:success`
66
+ # : The backup completed successfully.
67
+ # : Notification will be sent if `on_success` is `true`.
68
+ #
69
+ # `:warning`
70
+ # : The backup completed successfully, but warnings were logged.
71
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
72
+ #
73
+ # `:failure`
74
+ # : The backup operation failed.
75
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
76
+ #
77
+ def notify!(status)
78
+ status_data = status_data_for(status)
79
+ msg = message.call(model, status: status_data)
80
+ send_message(msg, status_data[:color])
81
+ end
82
+
83
+ def client_options
84
+ { api_version: @api_version }.tap do |h|
85
+ h[:server_url] = server_url if server_url
86
+ end
87
+ end
88
+
89
+ # Hipchat::Client will raise an error if unsuccessful.
90
+ def send_message(msg, color)
91
+ client = HipChat::Client.new(token, client_options)
92
+ rooms_to_notify.each do |room|
93
+ client[room].send(from, msg, color: color, notify: notify_users)
94
+ end
95
+ end
96
+
97
+ def rooms_to_notify
98
+ Array(rooms_notified).map { |r| r.split(",").map(&:strip) }.flatten
99
+ end
100
+
101
+ def status_data_for(status)
102
+ data = super(status)
103
+ data[:color] = status_color_for(status)
104
+ data
105
+ end
106
+
107
+ def status_color_for(status)
108
+ {
109
+ success: success_color,
110
+ warning: warning_color,
111
+ failure: failure_color
112
+ }[status]
113
+ end
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,114 @@
1
+ require "uri"
2
+
3
+ module Backup
4
+ module Notifier
5
+ class HttpPost < Base
6
+ ##
7
+ # URI to post notification to.
8
+ #
9
+ # URI scheme may be `http` or `https`.
10
+ #
11
+ # If Basic Authentication is needed, supply the `user:password` in the URI.
12
+ # e.g. 'https://user:pass@www.example.com/path'
13
+ #
14
+ # Port may also be supplied.
15
+ # e.g. 'http://www.example.com:8080/path'
16
+ attr_accessor :uri
17
+
18
+ ##
19
+ # Hash of additional HTTP headers to send.
20
+ #
21
+ # This notifier sets the following headers:
22
+ # { 'User-Agent' => "Backup/#{ Backup::VERSION }",
23
+ # 'Content-Type' => 'x-www-form-urlencoded' }
24
+ #
25
+ # 'Content-Type' may not be changed.
26
+ # 'User-Agent' may be overridden or omitted by setting it to +nil+.
27
+ # e.g. { 'Authorization' => 'my_auth_info', 'User-Agent' => nil }
28
+ attr_accessor :headers
29
+
30
+ ##
31
+ # Hash of additional POST parameters to send.
32
+ #
33
+ # This notifier will set two parameters:
34
+ # { 'status' => 'success|warning|failure',
35
+ # 'message' => '[Backup::(Success|Warning|Failure)] label (trigger)' }
36
+ #
37
+ # 'status' may not be changed.
38
+ # 'message' may be overridden or omitted by setting a +nil+ value.
39
+ # e.g. { 'auth_token' => 'my_token', 'message' => nil }
40
+ attr_accessor :params
41
+
42
+ ##
43
+ # Successful HTTP Status Code(s) that should be returned.
44
+ #
45
+ # This may be a single code or an Array of acceptable codes.
46
+ # e.g. [200, 201, 204]
47
+ #
48
+ # If any other response code is returned, the request will be retried
49
+ # using `max_retries` and `retry_waitsec`.
50
+ #
51
+ # Default: 200
52
+ attr_accessor :success_codes
53
+
54
+ ##
55
+ # Verify the server's certificate when using SSL.
56
+ #
57
+ # This will default to +true+ for most systems.
58
+ # It may be forced by setting to +true+, or disabled by setting to +false+.
59
+ attr_accessor :ssl_verify_peer
60
+
61
+ ##
62
+ # Path to a +cacert.pem+ file to use for +ssl_verify_peer+.
63
+ #
64
+ # This is provided (via Excon), but may be specified if needed.
65
+ attr_accessor :ssl_ca_file
66
+
67
+ def initialize(model, &block)
68
+ super
69
+ instance_eval(&block) if block_given?
70
+
71
+ @headers ||= {}
72
+ @params ||= {}
73
+ @success_codes ||= 200
74
+ end
75
+
76
+ private
77
+
78
+ ##
79
+ # Notify the user of the backup operation results.
80
+ #
81
+ # `status` indicates one of the following:
82
+ #
83
+ # `:success`
84
+ # : The backup completed successfully.
85
+ # : Notification will be sent if `on_success` is `true`.
86
+ #
87
+ # `:warning`
88
+ # : The backup completed successfully, but warnings were logged.
89
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
90
+ #
91
+ # `:failure`
92
+ # : The backup operation failed.
93
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
94
+ #
95
+ def notify!(status)
96
+ msg = message.call(model, status: status_data_for(status))
97
+
98
+ opts = {
99
+ headers: { "User-Agent" => "Backup/#{VERSION}" }
100
+ .merge(headers).reject { |_, value| value.nil? }
101
+ .merge("Content-Type" => "application/x-www-form-urlencoded"),
102
+ body: URI.encode_www_form({ "message" => msg }
103
+ .merge(params).reject { |_, value| value.nil? }
104
+ .merge("status" => status.to_s)),
105
+ expects: success_codes # raise error if unsuccessful
106
+ }
107
+ opts[:ssl_verify_peer] = ssl_verify_peer unless ssl_verify_peer.nil?
108
+ opts[:ssl_ca_file] = ssl_ca_file if ssl_ca_file
109
+
110
+ Excon.post(uri, opts)
111
+ end
112
+ end
113
+ end
114
+ end
@@ -0,0 +1,246 @@
1
+ require "mail"
2
+
3
+ module Backup
4
+ module Notifier
5
+ class Mail < Base
6
+ ##
7
+ # Mail delivery method to be used by the Mail gem.
8
+ #
9
+ # Supported methods:
10
+ #
11
+ # [:smtp - ::Mail::SMTP (default)]
12
+ # Settings used by this method:
13
+ # {#address}, {#port}, {#domain}, {#user_name}, {#password},
14
+ # {#authentication}, {#encryption}, {#openssl_verify_mode}
15
+ #
16
+ # [:sendmail - ::Mail::Sendmail]
17
+ # Settings used by this method:
18
+ # {#sendmail_args}
19
+ #
20
+ # [:exim - ::Mail::Exim]
21
+ # Settings used by this method:
22
+ # {#exim_args}
23
+ #
24
+ # [:file - ::Mail::FileDelivery]
25
+ # Settings used by this method:
26
+ # {#mail_folder}
27
+ #
28
+ attr_accessor :delivery_method
29
+
30
+ ##
31
+ # Sender Email Address
32
+ attr_accessor :from
33
+
34
+ ##
35
+ # Receiver Email Address
36
+ attr_accessor :to
37
+
38
+ ##
39
+ # CC receiver Email Address
40
+ attr_accessor :cc
41
+
42
+ ##
43
+ # BCC receiver Email Address
44
+ attr_accessor :bcc
45
+
46
+ ##
47
+ # Set reply to email address
48
+ attr_accessor :reply_to
49
+
50
+ ##
51
+ # SMTP Server Address
52
+ attr_accessor :address
53
+
54
+ ##
55
+ # SMTP Server Port
56
+ attr_accessor :port
57
+
58
+ ##
59
+ # Your domain (if applicable)
60
+ attr_accessor :domain
61
+
62
+ ##
63
+ # SMTP Server Username (sender email's credentials)
64
+ attr_accessor :user_name
65
+
66
+ ##
67
+ # SMTP Server Password (sender email's credentials)
68
+ attr_accessor :password
69
+
70
+ ##
71
+ # Authentication type
72
+ #
73
+ # Acceptable values: +:plain+, +:login+, +:cram_md5+
74
+ attr_accessor :authentication
75
+
76
+ ##
77
+ # Set the method of encryption to be used for the +SMTP+ connection.
78
+ #
79
+ # [:starttls (default)]
80
+ # Use +STARTTLS+ to upgrade the connection to a +SSL/TLS+ connection.
81
+ #
82
+ # [:tls or :ssl]
83
+ # Use a +SSL/TLS+ connection.
84
+ #
85
+ # [:none]
86
+ # No encryption will be used.
87
+ attr_accessor :encryption
88
+
89
+ ##
90
+ # OpenSSL Verify Mode
91
+ #
92
+ # Valid modes: +:none+, +:peer+, +:client_once+, +:fail_if_no_peer_cert+
93
+ # See +OpenSSL::SSL+ for details.
94
+ #
95
+ # Use +:none+ for a self-signed and/or wildcard certificate
96
+ attr_accessor :openssl_verify_mode
97
+
98
+ ##
99
+ # Optional arguments to pass to `sendmail`
100
+ #
101
+ # Note that this will override the defaults set by the Mail gem
102
+ # (currently: '-i'). So, if set here, be sure to set all the arguments
103
+ # you require.
104
+ #
105
+ # Example: '-i -X/tmp/traffic.log'
106
+ attr_accessor :sendmail_args
107
+
108
+ ##
109
+ # Optional arguments to pass to `exim`
110
+ #
111
+ # Note that this will override the defaults set by the Mail gem
112
+ # (currently: '-i -t') So, if set here, be sure to set all the arguments
113
+ # you require.
114
+ #
115
+ # Example: '-i -t -X/tmp/traffic.log'
116
+ attr_accessor :exim_args
117
+
118
+ ##
119
+ # Folder where mail will be kept when using the `:file` `delivery_method`.
120
+ #
121
+ # Default location is '$HOME/Backup/emails'
122
+ attr_accessor :mail_folder
123
+
124
+ ##
125
+ # Array of statuses for which the log file should be attached.
126
+ #
127
+ # Available statuses are: `:success`, `:warning` and `:failure`.
128
+ # Default: [:warning, :failure]
129
+ attr_accessor :send_log_on
130
+
131
+ def initialize(model, &block)
132
+ super
133
+ instance_eval(&block) if block_given?
134
+
135
+ @send_log_on ||= [:warning, :failure]
136
+ @encryption ||= :starttls
137
+ end
138
+
139
+ private
140
+
141
+ ##
142
+ # Notify the user of the backup operation results.
143
+ #
144
+ # `status` indicates one of the following:
145
+ #
146
+ # `:success`
147
+ # : The backup completed successfully.
148
+ # : Notification will be sent if `on_success` is `true`.
149
+ #
150
+ # `:warning`
151
+ # : The backup completed successfully, but warnings were logged.
152
+ # : Notification will be sent, including a copy of the current
153
+ # : backup log, if `on_warning` or `on_success` is `true`.
154
+ #
155
+ # `:failure`
156
+ # : The backup operation failed.
157
+ # : Notification will be sent, including a copy of the current
158
+ # : backup log, if `on_failure` is `true`.
159
+ #
160
+ def notify!(status)
161
+ email = new_email
162
+ email.subject = message.call(model, status: status_data_for(status))
163
+
164
+ send_log = send_log_on.include?(status)
165
+ template = Backup::Template.new(model: model, send_log: send_log)
166
+ email.body = template.result(sprintf("notifier/mail/%s.erb", status.to_s))
167
+
168
+ if send_log
169
+ email.convert_to_multipart
170
+ email.attachments["#{model.time}.#{model.trigger}.log"] = {
171
+ mime_type: "text/plain;",
172
+ content: Logger.messages.map(&:formatted_lines).flatten.join("\n")
173
+ }
174
+ end
175
+
176
+ email.deliver! # raise error if unsuccessful
177
+ end
178
+
179
+ ##
180
+ # Configures the Mail gem by setting the defaults.
181
+ # Creates and returns a new email, based on the @delivery_method used.
182
+ def new_email
183
+ method = %w(smtp sendmail exim file test)
184
+ .index(@delivery_method.to_s) ? @delivery_method.to_s : "smtp"
185
+
186
+ options =
187
+ case method
188
+ when "smtp"
189
+ opts = {
190
+ address: @address,
191
+ port: @port,
192
+ user_name: @user_name,
193
+ password: @password,
194
+ authentication: @authentication,
195
+ enable_starttls_auto: @encryption == :starttls,
196
+ openssl_verify_mode: @openssl_verify_mode,
197
+ ssl: @encryption == :ssl,
198
+ tls: @encryption == :tls
199
+ }
200
+
201
+ # Don't override default domain setting if domain not applicable.
202
+ # ref https://github.com/mikel/mail/blob/2.6.3/lib/mail/network/delivery_methods/smtp.rb#L82
203
+ opts[:domain] = @domain if @domain
204
+ opts
205
+ when "sendmail"
206
+ opts = {}
207
+ opts[:location] = utility(:sendmail)
208
+ opts[:arguments] = @sendmail_args if @sendmail_args
209
+ opts
210
+ when "exim"
211
+ opts = {}
212
+ opts[:location] = utility(:exim)
213
+ opts[:arguments] = @exim_args if @exim_args
214
+ opts
215
+ when "file"
216
+ @mail_folder ||= File.join(Config.root_path, "emails")
217
+ { location: File.expand_path(@mail_folder) }
218
+ when "test" then {}
219
+ end
220
+
221
+ email = ::Mail.new
222
+ email.delivery_method method.to_sym, options
223
+ email.to = to
224
+ email.from = from
225
+ email.cc = cc
226
+ email.bcc = bcc
227
+ email.reply_to = reply_to
228
+ email
229
+ end
230
+ end
231
+ end
232
+ end
233
+
234
+ # Patch mail v2.5.4 Exim delivery method
235
+ # https://github.com/backup/backup/issues/446
236
+ # https://github.com/mikel/mail/pull/546
237
+ module Mail
238
+ class Exim
239
+ def self.call(path, arguments, _destinations, encoded_message)
240
+ popen "#{path} #{arguments}" do |io|
241
+ io.puts encoded_message.to_lf
242
+ io.flush
243
+ end
244
+ end
245
+ end
246
+ end