cm-backup 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +20 -0
  3. data/bin/backup +5 -0
  4. data/lib/backup.rb +144 -0
  5. data/lib/backup/archive.rb +170 -0
  6. data/lib/backup/binder.rb +22 -0
  7. data/lib/backup/cleaner.rb +116 -0
  8. data/lib/backup/cli.rb +374 -0
  9. data/lib/backup/cloud_io/base.rb +41 -0
  10. data/lib/backup/cloud_io/cloud_files.rb +298 -0
  11. data/lib/backup/cloud_io/s3.rb +260 -0
  12. data/lib/backup/compressor/base.rb +35 -0
  13. data/lib/backup/compressor/bzip2.rb +39 -0
  14. data/lib/backup/compressor/custom.rb +53 -0
  15. data/lib/backup/compressor/gzip.rb +74 -0
  16. data/lib/backup/config.rb +119 -0
  17. data/lib/backup/config/dsl.rb +103 -0
  18. data/lib/backup/config/helpers.rb +143 -0
  19. data/lib/backup/database/base.rb +85 -0
  20. data/lib/backup/database/mongodb.rb +187 -0
  21. data/lib/backup/database/mysql.rb +192 -0
  22. data/lib/backup/database/openldap.rb +95 -0
  23. data/lib/backup/database/postgresql.rb +133 -0
  24. data/lib/backup/database/redis.rb +179 -0
  25. data/lib/backup/database/riak.rb +82 -0
  26. data/lib/backup/database/sqlite.rb +57 -0
  27. data/lib/backup/encryptor/base.rb +29 -0
  28. data/lib/backup/encryptor/gpg.rb +747 -0
  29. data/lib/backup/encryptor/open_ssl.rb +77 -0
  30. data/lib/backup/errors.rb +58 -0
  31. data/lib/backup/logger.rb +199 -0
  32. data/lib/backup/logger/console.rb +51 -0
  33. data/lib/backup/logger/fog_adapter.rb +29 -0
  34. data/lib/backup/logger/logfile.rb +133 -0
  35. data/lib/backup/logger/syslog.rb +116 -0
  36. data/lib/backup/model.rb +479 -0
  37. data/lib/backup/notifier/base.rb +128 -0
  38. data/lib/backup/notifier/campfire.rb +63 -0
  39. data/lib/backup/notifier/command.rb +102 -0
  40. data/lib/backup/notifier/datadog.rb +107 -0
  41. data/lib/backup/notifier/flowdock.rb +103 -0
  42. data/lib/backup/notifier/hipchat.rb +118 -0
  43. data/lib/backup/notifier/http_post.rb +117 -0
  44. data/lib/backup/notifier/mail.rb +249 -0
  45. data/lib/backup/notifier/nagios.rb +69 -0
  46. data/lib/backup/notifier/pagerduty.rb +81 -0
  47. data/lib/backup/notifier/prowl.rb +68 -0
  48. data/lib/backup/notifier/pushover.rb +74 -0
  49. data/lib/backup/notifier/ses.rb +105 -0
  50. data/lib/backup/notifier/slack.rb +148 -0
  51. data/lib/backup/notifier/twitter.rb +58 -0
  52. data/lib/backup/notifier/zabbix.rb +63 -0
  53. data/lib/backup/package.rb +55 -0
  54. data/lib/backup/packager.rb +107 -0
  55. data/lib/backup/pipeline.rb +124 -0
  56. data/lib/backup/splitter.rb +76 -0
  57. data/lib/backup/storage/base.rb +69 -0
  58. data/lib/backup/storage/cloud_files.rb +158 -0
  59. data/lib/backup/storage/cycler.rb +75 -0
  60. data/lib/backup/storage/dropbox.rb +212 -0
  61. data/lib/backup/storage/ftp.rb +112 -0
  62. data/lib/backup/storage/local.rb +64 -0
  63. data/lib/backup/storage/qiniu.rb +65 -0
  64. data/lib/backup/storage/rsync.rb +248 -0
  65. data/lib/backup/storage/s3.rb +156 -0
  66. data/lib/backup/storage/scp.rb +67 -0
  67. data/lib/backup/storage/sftp.rb +82 -0
  68. data/lib/backup/syncer/base.rb +70 -0
  69. data/lib/backup/syncer/cloud/base.rb +179 -0
  70. data/lib/backup/syncer/cloud/cloud_files.rb +83 -0
  71. data/lib/backup/syncer/cloud/local_file.rb +100 -0
  72. data/lib/backup/syncer/cloud/s3.rb +110 -0
  73. data/lib/backup/syncer/rsync/base.rb +54 -0
  74. data/lib/backup/syncer/rsync/local.rb +31 -0
  75. data/lib/backup/syncer/rsync/pull.rb +51 -0
  76. data/lib/backup/syncer/rsync/push.rb +205 -0
  77. data/lib/backup/template.rb +46 -0
  78. data/lib/backup/utilities.rb +224 -0
  79. data/lib/backup/version.rb +5 -0
  80. data/templates/cli/archive +28 -0
  81. data/templates/cli/compressor/bzip2 +4 -0
  82. data/templates/cli/compressor/custom +7 -0
  83. data/templates/cli/compressor/gzip +4 -0
  84. data/templates/cli/config +123 -0
  85. data/templates/cli/databases/mongodb +15 -0
  86. data/templates/cli/databases/mysql +18 -0
  87. data/templates/cli/databases/openldap +24 -0
  88. data/templates/cli/databases/postgresql +16 -0
  89. data/templates/cli/databases/redis +16 -0
  90. data/templates/cli/databases/riak +17 -0
  91. data/templates/cli/databases/sqlite +11 -0
  92. data/templates/cli/encryptor/gpg +27 -0
  93. data/templates/cli/encryptor/openssl +9 -0
  94. data/templates/cli/model +26 -0
  95. data/templates/cli/notifier/zabbix +15 -0
  96. data/templates/cli/notifiers/campfire +12 -0
  97. data/templates/cli/notifiers/command +32 -0
  98. data/templates/cli/notifiers/datadog +57 -0
  99. data/templates/cli/notifiers/flowdock +16 -0
  100. data/templates/cli/notifiers/hipchat +16 -0
  101. data/templates/cli/notifiers/http_post +32 -0
  102. data/templates/cli/notifiers/mail +24 -0
  103. data/templates/cli/notifiers/nagios +13 -0
  104. data/templates/cli/notifiers/pagerduty +12 -0
  105. data/templates/cli/notifiers/prowl +11 -0
  106. data/templates/cli/notifiers/pushover +11 -0
  107. data/templates/cli/notifiers/ses +15 -0
  108. data/templates/cli/notifiers/slack +22 -0
  109. data/templates/cli/notifiers/twitter +13 -0
  110. data/templates/cli/splitter +7 -0
  111. data/templates/cli/storages/cloud_files +11 -0
  112. data/templates/cli/storages/dropbox +20 -0
  113. data/templates/cli/storages/ftp +13 -0
  114. data/templates/cli/storages/local +8 -0
  115. data/templates/cli/storages/qiniu +12 -0
  116. data/templates/cli/storages/rsync +17 -0
  117. data/templates/cli/storages/s3 +16 -0
  118. data/templates/cli/storages/scp +15 -0
  119. data/templates/cli/storages/sftp +15 -0
  120. data/templates/cli/syncers/cloud_files +22 -0
  121. data/templates/cli/syncers/rsync_local +20 -0
  122. data/templates/cli/syncers/rsync_pull +28 -0
  123. data/templates/cli/syncers/rsync_push +28 -0
  124. data/templates/cli/syncers/s3 +27 -0
  125. data/templates/general/links +3 -0
  126. data/templates/general/version.erb +2 -0
  127. data/templates/notifier/mail/failure.erb +16 -0
  128. data/templates/notifier/mail/success.erb +16 -0
  129. data/templates/notifier/mail/warning.erb +16 -0
  130. data/templates/storage/dropbox/authorization_url.erb +6 -0
  131. data/templates/storage/dropbox/authorized.erb +4 -0
  132. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  133. metadata +1077 -0
@@ -0,0 +1,118 @@
1
+ # encoding: utf-8
2
+ require 'hipchat'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class Hipchat < Base
7
+
8
+ ##
9
+ # The Hipchat API token
10
+ attr_accessor :token
11
+
12
+ ##
13
+ # The Hipchat API version
14
+ # Either 'v1' or 'v2' (default is 'v1')
15
+ attr_accessor :api_version
16
+
17
+ ##
18
+ # Who the notification should appear from
19
+ attr_accessor :from
20
+
21
+ ##
22
+ # Custom server URL
23
+ attr_accessor :server_url
24
+
25
+ ##
26
+ # The rooms that should be notified
27
+ attr_accessor :rooms_notified
28
+
29
+ ##
30
+ # Notify users in the room
31
+ attr_accessor :notify_users
32
+
33
+ ##
34
+ # The background color of a success message.
35
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
36
+ attr_accessor :success_color
37
+
38
+ ##
39
+ # The background color of a warning message.
40
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
41
+ attr_accessor :warning_color
42
+
43
+ ##
44
+ # The background color of an error message.
45
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
46
+ attr_accessor :failure_color
47
+
48
+ def initialize(model, &block)
49
+ super
50
+ instance_eval(&block) if block_given?
51
+
52
+ @notify_users ||= false
53
+ @rooms_notified ||= []
54
+ @success_color ||= 'yellow'
55
+ @warning_color ||= 'yellow'
56
+ @failure_color ||= 'yellow'
57
+ @api_version ||= 'v1'
58
+ end
59
+
60
+ private
61
+
62
+ ##
63
+ # Notify the user of the backup operation results.
64
+ #
65
+ # `status` indicates one of the following:
66
+ #
67
+ # `:success`
68
+ # : The backup completed successfully.
69
+ # : Notification will be sent if `on_success` is `true`.
70
+ #
71
+ # `:warning`
72
+ # : The backup completed successfully, but warnings were logged.
73
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
74
+ #
75
+ # `:failure`
76
+ # : The backup operation failed.
77
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
78
+ #
79
+ def notify!(status)
80
+ status_data = status_data_for(status)
81
+ msg = message.call(model, :status => status_data)
82
+ send_message(msg, status_data[:color])
83
+ end
84
+
85
+ def client_options
86
+ { api_version: @api_version }.tap do |h|
87
+ h[:server_url] = server_url if server_url
88
+ end
89
+ end
90
+
91
+ # Hipchat::Client will raise an error if unsuccessful.
92
+ def send_message(msg, color)
93
+ client = HipChat::Client.new(token, client_options)
94
+ rooms_to_notify.each do |room|
95
+ client[room].send(from, msg, :color => color, :notify => notify_users)
96
+ end
97
+ end
98
+
99
+ def rooms_to_notify
100
+ Array(rooms_notified).map {|r| r.split(',').map(&:strip) }.flatten
101
+ end
102
+
103
+ def status_data_for(status)
104
+ data = super(status)
105
+ data[:color] = status_color_for(status)
106
+ data
107
+ end
108
+
109
+ def status_color_for(status)
110
+ {
111
+ :success => success_color,
112
+ :warning => warning_color,
113
+ :failure => failure_color
114
+ }[status]
115
+ end
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,117 @@
1
+ # encoding: utf-8
2
+ require 'uri'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class HttpPost < Base
7
+
8
+ ##
9
+ # URI to post notification to.
10
+ #
11
+ # URI scheme may be `http` or `https`.
12
+ #
13
+ # If Basic Authentication is needed, supply the `user:password` in the URI.
14
+ # e.g. 'https://user:pass@www.example.com/path'
15
+ #
16
+ # Port may also be supplied.
17
+ # e.g. 'http://www.example.com:8080/path'
18
+ attr_accessor :uri
19
+
20
+ ##
21
+ # Hash of additional HTTP headers to send.
22
+ #
23
+ # This notifier sets the following headers:
24
+ # { 'User-Agent' => "Backup/#{ Backup::VERSION }",
25
+ # 'Content-Type' => 'x-www-form-urlencoded' }
26
+ #
27
+ # 'Content-Type' may not be changed.
28
+ # 'User-Agent' may be overridden or omitted by setting it to +nil+.
29
+ # e.g. { 'Authorization' => 'my_auth_info', 'User-Agent' => nil }
30
+ attr_accessor :headers
31
+
32
+ ##
33
+ # Hash of additional POST parameters to send.
34
+ #
35
+ # This notifier will set two parameters:
36
+ # { 'status' => 'success|warning|failure',
37
+ # 'message' => '[Backup::(Success|Warning|Failure)] label (trigger)' }
38
+ #
39
+ # 'status' may not be changed.
40
+ # 'message' may be overridden or omitted by setting a +nil+ value.
41
+ # e.g. { 'auth_token' => 'my_token', 'message' => nil }
42
+ attr_accessor :params
43
+
44
+ ##
45
+ # Successful HTTP Status Code(s) that should be returned.
46
+ #
47
+ # This may be a single code or an Array of acceptable codes.
48
+ # e.g. [200, 201, 204]
49
+ #
50
+ # If any other response code is returned, the request will be retried
51
+ # using `max_retries` and `retry_waitsec`.
52
+ #
53
+ # Default: 200
54
+ attr_accessor :success_codes
55
+
56
+ ##
57
+ # Verify the server's certificate when using SSL.
58
+ #
59
+ # This will default to +true+ for most systems.
60
+ # It may be forced by setting to +true+, or disabled by setting to +false+.
61
+ attr_accessor :ssl_verify_peer
62
+
63
+ ##
64
+ # Path to a +cacert.pem+ file to use for +ssl_verify_peer+.
65
+ #
66
+ # This is provided (via Excon), but may be specified if needed.
67
+ attr_accessor :ssl_ca_file
68
+
69
+ def initialize(model, &block)
70
+ super
71
+ instance_eval(&block) if block_given?
72
+
73
+ @headers ||= {}
74
+ @params ||= {}
75
+ @success_codes ||= 200
76
+ end
77
+
78
+ private
79
+
80
+ ##
81
+ # Notify the user of the backup operation results.
82
+ #
83
+ # `status` indicates one of the following:
84
+ #
85
+ # `:success`
86
+ # : The backup completed successfully.
87
+ # : Notification will be sent if `on_success` is `true`.
88
+ #
89
+ # `:warning`
90
+ # : The backup completed successfully, but warnings were logged.
91
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
92
+ #
93
+ # `:failure`
94
+ # : The backup operation failed.
95
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
96
+ #
97
+ def notify!(status)
98
+ msg = message.call(model, :status => status_data_for(status))
99
+
100
+ opts = {
101
+ :headers => { 'User-Agent' => "Backup/#{ VERSION }" }.
102
+ merge(headers).reject {|k,v| v.nil? }.
103
+ merge('Content-Type' => 'application/x-www-form-urlencoded'),
104
+ :body => URI.encode_www_form({ 'message' => msg }.
105
+ merge(params).reject {|k,v| v.nil? }.
106
+ merge('status' => status.to_s)),
107
+ :expects => success_codes # raise error if unsuccessful
108
+ }
109
+ opts.merge!(:ssl_verify_peer => ssl_verify_peer) unless ssl_verify_peer.nil?
110
+ opts.merge!(:ssl_ca_file => ssl_ca_file) if ssl_ca_file
111
+
112
+ Excon.post(uri, opts)
113
+ end
114
+
115
+ end
116
+ end
117
+ end
@@ -0,0 +1,249 @@
1
+ # encoding: utf-8
2
+ require 'mail'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class Mail < Base
7
+
8
+ ##
9
+ # Mail delivery method to be used by the Mail gem.
10
+ #
11
+ # Supported methods:
12
+ #
13
+ # [:smtp - ::Mail::SMTP (default)]
14
+ # Settings used by this method:
15
+ # {#address}, {#port}, {#domain}, {#user_name}, {#password},
16
+ # {#authentication}, {#encryption}, {#openssl_verify_mode}
17
+ #
18
+ # [:sendmail - ::Mail::Sendmail]
19
+ # Settings used by this method:
20
+ # {#sendmail_args}
21
+ #
22
+ # [:exim - ::Mail::Exim]
23
+ # Settings used by this method:
24
+ # {#exim_args}
25
+ #
26
+ # [:file - ::Mail::FileDelivery]
27
+ # Settings used by this method:
28
+ # {#mail_folder}
29
+ #
30
+ attr_accessor :delivery_method
31
+
32
+ ##
33
+ # Sender Email Address
34
+ attr_accessor :from
35
+
36
+ ##
37
+ # Receiver Email Address
38
+ attr_accessor :to
39
+
40
+ ##
41
+ # CC receiver Email Address
42
+ attr_accessor :cc
43
+
44
+ ##
45
+ # BCC receiver Email Address
46
+ attr_accessor :bcc
47
+
48
+ ##
49
+ # Set reply to email address
50
+ attr_accessor :reply_to
51
+
52
+ ##
53
+ # SMTP Server Address
54
+ attr_accessor :address
55
+
56
+ ##
57
+ # SMTP Server Port
58
+ attr_accessor :port
59
+
60
+ ##
61
+ # Your domain (if applicable)
62
+ attr_accessor :domain
63
+
64
+ ##
65
+ # SMTP Server Username (sender email's credentials)
66
+ attr_accessor :user_name
67
+
68
+ ##
69
+ # SMTP Server Password (sender email's credentials)
70
+ attr_accessor :password
71
+
72
+ ##
73
+ # Authentication type
74
+ #
75
+ # Acceptable values: +:plain+, +:login+, +:cram_md5+
76
+ attr_accessor :authentication
77
+
78
+ ##
79
+ # Set the method of encryption to be used for the +SMTP+ connection.
80
+ #
81
+ # [:starttls (default)]
82
+ # Use +STARTTLS+ to upgrade the connection to a +SSL/TLS+ connection.
83
+ #
84
+ # [:tls or :ssl]
85
+ # Use a +SSL/TLS+ connection.
86
+ #
87
+ # [:none]
88
+ # No encryption will be used.
89
+ attr_accessor :encryption
90
+
91
+ ##
92
+ # OpenSSL Verify Mode
93
+ #
94
+ # Valid modes: +:none+, +:peer+, +:client_once+, +:fail_if_no_peer_cert+
95
+ # See +OpenSSL::SSL+ for details.
96
+ #
97
+ # Use +:none+ for a self-signed and/or wildcard certificate
98
+ attr_accessor :openssl_verify_mode
99
+
100
+ ##
101
+ # Optional arguments to pass to `sendmail`
102
+ #
103
+ # Note that this will override the defaults set by the Mail gem
104
+ # (currently: '-i'). So, if set here, be sure to set all the arguments
105
+ # you require.
106
+ #
107
+ # Example: '-i -X/tmp/traffic.log'
108
+ attr_accessor :sendmail_args
109
+
110
+ ##
111
+ # Optional arguments to pass to `exim`
112
+ #
113
+ # Note that this will override the defaults set by the Mail gem
114
+ # (currently: '-i -t') So, if set here, be sure to set all the arguments
115
+ # you require.
116
+ #
117
+ # Example: '-i -t -X/tmp/traffic.log'
118
+ attr_accessor :exim_args
119
+
120
+ ##
121
+ # Folder where mail will be kept when using the `:file` `delivery_method`.
122
+ #
123
+ # Default location is '$HOME/Backup/emails'
124
+ attr_accessor :mail_folder
125
+
126
+ ##
127
+ # Array of statuses for which the log file should be attached.
128
+ #
129
+ # Available statuses are: `:success`, `:warning` and `:failure`.
130
+ # Default: [:warning, :failure]
131
+ attr_accessor :send_log_on
132
+
133
+ def initialize(model, &block)
134
+ super
135
+ instance_eval(&block) if block_given?
136
+
137
+ @send_log_on ||= [:warning, :failure]
138
+ @encryption ||= :starttls
139
+ end
140
+
141
+ private
142
+
143
+ ##
144
+ # Notify the user of the backup operation results.
145
+ #
146
+ # `status` indicates one of the following:
147
+ #
148
+ # `:success`
149
+ # : The backup completed successfully.
150
+ # : Notification will be sent if `on_success` is `true`.
151
+ #
152
+ # `:warning`
153
+ # : The backup completed successfully, but warnings were logged.
154
+ # : Notification will be sent, including a copy of the current
155
+ # : backup log, if `on_warning` or `on_success` is `true`.
156
+ #
157
+ # `:failure`
158
+ # : The backup operation failed.
159
+ # : Notification will be sent, including a copy of the current
160
+ # : backup log, if `on_failure` is `true`.
161
+ #
162
+ def notify!(status)
163
+ email = new_email
164
+ email.subject = message.call(model, :status => status_data_for(status))
165
+
166
+ send_log = send_log_on.include?(status)
167
+ template = Backup::Template.new({ :model => model, :send_log => send_log })
168
+ email.body = template.result('notifier/mail/%s.erb' % status.to_s)
169
+
170
+ if send_log
171
+ email.convert_to_multipart
172
+ email.attachments["#{ model.time }.#{ model.trigger }.log"] = {
173
+ :mime_type => 'text/plain;',
174
+ :content => Logger.messages.map(&:formatted_lines).flatten.join("\n")
175
+ }
176
+ end
177
+
178
+ email.deliver! # raise error if unsuccessful
179
+ end
180
+
181
+ ##
182
+ # Configures the Mail gem by setting the defaults.
183
+ # Creates and returns a new email, based on the @delivery_method used.
184
+ def new_email
185
+ method = %w{ smtp sendmail exim file test }.
186
+ index(@delivery_method.to_s) ? @delivery_method.to_s : 'smtp'
187
+
188
+ options =
189
+ case method
190
+ when 'smtp'
191
+ opts = {
192
+ :address => @address,
193
+ :port => @port,
194
+ :user_name => @user_name,
195
+ :password => @password,
196
+ :authentication => @authentication,
197
+ :enable_starttls_auto => @encryption == :starttls,
198
+ :openssl_verify_mode => @openssl_verify_mode,
199
+ :ssl => @encryption == :ssl,
200
+ :tls => @encryption == :tls
201
+ }
202
+
203
+ # Don't override default domain setting if domain not applicable.
204
+ # ref https://github.com/mikel/mail/blob/2.6.3/lib/mail/network/delivery_methods/smtp.rb#L82
205
+ opts[:domain] = @domain if @domain
206
+ opts
207
+ when 'sendmail'
208
+ opts = {}
209
+ opts.merge!(:location => utility(:sendmail))
210
+ opts.merge!(:arguments => @sendmail_args) if @sendmail_args
211
+ opts
212
+ when 'exim'
213
+ opts = {}
214
+ opts.merge!(:location => utility(:exim))
215
+ opts.merge!(:arguments => @exim_args) if @exim_args
216
+ opts
217
+ when 'file'
218
+ @mail_folder ||= File.join(Config.root_path, 'emails')
219
+ { :location => File.expand_path(@mail_folder) }
220
+ when 'test' then {}
221
+ end
222
+
223
+ email = ::Mail.new
224
+ email.delivery_method method.to_sym, options
225
+ email.to = to
226
+ email.from = from
227
+ email.cc = cc
228
+ email.bcc = bcc
229
+ email.reply_to = reply_to
230
+ email
231
+ end
232
+
233
+ end
234
+ end
235
+ end
236
+
237
+ # Patch mail v2.5.4 Exim delivery method
238
+ # https://github.com/backup/backup/issues/446
239
+ # https://github.com/mikel/mail/pull/546
240
+ module Mail
241
+ class Exim
242
+ def self.call(path, arguments, destinations, encoded_message)
243
+ popen "#{path} #{arguments}" do |io|
244
+ io.puts encoded_message.to_lf
245
+ io.flush
246
+ end
247
+ end
248
+ end
249
+ end