cm-backup 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (133) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +20 -0
  3. data/bin/backup +5 -0
  4. data/lib/backup.rb +144 -0
  5. data/lib/backup/archive.rb +170 -0
  6. data/lib/backup/binder.rb +22 -0
  7. data/lib/backup/cleaner.rb +116 -0
  8. data/lib/backup/cli.rb +374 -0
  9. data/lib/backup/cloud_io/base.rb +41 -0
  10. data/lib/backup/cloud_io/cloud_files.rb +298 -0
  11. data/lib/backup/cloud_io/s3.rb +260 -0
  12. data/lib/backup/compressor/base.rb +35 -0
  13. data/lib/backup/compressor/bzip2.rb +39 -0
  14. data/lib/backup/compressor/custom.rb +53 -0
  15. data/lib/backup/compressor/gzip.rb +74 -0
  16. data/lib/backup/config.rb +119 -0
  17. data/lib/backup/config/dsl.rb +103 -0
  18. data/lib/backup/config/helpers.rb +143 -0
  19. data/lib/backup/database/base.rb +85 -0
  20. data/lib/backup/database/mongodb.rb +187 -0
  21. data/lib/backup/database/mysql.rb +192 -0
  22. data/lib/backup/database/openldap.rb +95 -0
  23. data/lib/backup/database/postgresql.rb +133 -0
  24. data/lib/backup/database/redis.rb +179 -0
  25. data/lib/backup/database/riak.rb +82 -0
  26. data/lib/backup/database/sqlite.rb +57 -0
  27. data/lib/backup/encryptor/base.rb +29 -0
  28. data/lib/backup/encryptor/gpg.rb +747 -0
  29. data/lib/backup/encryptor/open_ssl.rb +77 -0
  30. data/lib/backup/errors.rb +58 -0
  31. data/lib/backup/logger.rb +199 -0
  32. data/lib/backup/logger/console.rb +51 -0
  33. data/lib/backup/logger/fog_adapter.rb +29 -0
  34. data/lib/backup/logger/logfile.rb +133 -0
  35. data/lib/backup/logger/syslog.rb +116 -0
  36. data/lib/backup/model.rb +479 -0
  37. data/lib/backup/notifier/base.rb +128 -0
  38. data/lib/backup/notifier/campfire.rb +63 -0
  39. data/lib/backup/notifier/command.rb +102 -0
  40. data/lib/backup/notifier/datadog.rb +107 -0
  41. data/lib/backup/notifier/flowdock.rb +103 -0
  42. data/lib/backup/notifier/hipchat.rb +118 -0
  43. data/lib/backup/notifier/http_post.rb +117 -0
  44. data/lib/backup/notifier/mail.rb +249 -0
  45. data/lib/backup/notifier/nagios.rb +69 -0
  46. data/lib/backup/notifier/pagerduty.rb +81 -0
  47. data/lib/backup/notifier/prowl.rb +68 -0
  48. data/lib/backup/notifier/pushover.rb +74 -0
  49. data/lib/backup/notifier/ses.rb +105 -0
  50. data/lib/backup/notifier/slack.rb +148 -0
  51. data/lib/backup/notifier/twitter.rb +58 -0
  52. data/lib/backup/notifier/zabbix.rb +63 -0
  53. data/lib/backup/package.rb +55 -0
  54. data/lib/backup/packager.rb +107 -0
  55. data/lib/backup/pipeline.rb +124 -0
  56. data/lib/backup/splitter.rb +76 -0
  57. data/lib/backup/storage/base.rb +69 -0
  58. data/lib/backup/storage/cloud_files.rb +158 -0
  59. data/lib/backup/storage/cycler.rb +75 -0
  60. data/lib/backup/storage/dropbox.rb +212 -0
  61. data/lib/backup/storage/ftp.rb +112 -0
  62. data/lib/backup/storage/local.rb +64 -0
  63. data/lib/backup/storage/qiniu.rb +65 -0
  64. data/lib/backup/storage/rsync.rb +248 -0
  65. data/lib/backup/storage/s3.rb +156 -0
  66. data/lib/backup/storage/scp.rb +67 -0
  67. data/lib/backup/storage/sftp.rb +82 -0
  68. data/lib/backup/syncer/base.rb +70 -0
  69. data/lib/backup/syncer/cloud/base.rb +179 -0
  70. data/lib/backup/syncer/cloud/cloud_files.rb +83 -0
  71. data/lib/backup/syncer/cloud/local_file.rb +100 -0
  72. data/lib/backup/syncer/cloud/s3.rb +110 -0
  73. data/lib/backup/syncer/rsync/base.rb +54 -0
  74. data/lib/backup/syncer/rsync/local.rb +31 -0
  75. data/lib/backup/syncer/rsync/pull.rb +51 -0
  76. data/lib/backup/syncer/rsync/push.rb +205 -0
  77. data/lib/backup/template.rb +46 -0
  78. data/lib/backup/utilities.rb +224 -0
  79. data/lib/backup/version.rb +5 -0
  80. data/templates/cli/archive +28 -0
  81. data/templates/cli/compressor/bzip2 +4 -0
  82. data/templates/cli/compressor/custom +7 -0
  83. data/templates/cli/compressor/gzip +4 -0
  84. data/templates/cli/config +123 -0
  85. data/templates/cli/databases/mongodb +15 -0
  86. data/templates/cli/databases/mysql +18 -0
  87. data/templates/cli/databases/openldap +24 -0
  88. data/templates/cli/databases/postgresql +16 -0
  89. data/templates/cli/databases/redis +16 -0
  90. data/templates/cli/databases/riak +17 -0
  91. data/templates/cli/databases/sqlite +11 -0
  92. data/templates/cli/encryptor/gpg +27 -0
  93. data/templates/cli/encryptor/openssl +9 -0
  94. data/templates/cli/model +26 -0
  95. data/templates/cli/notifier/zabbix +15 -0
  96. data/templates/cli/notifiers/campfire +12 -0
  97. data/templates/cli/notifiers/command +32 -0
  98. data/templates/cli/notifiers/datadog +57 -0
  99. data/templates/cli/notifiers/flowdock +16 -0
  100. data/templates/cli/notifiers/hipchat +16 -0
  101. data/templates/cli/notifiers/http_post +32 -0
  102. data/templates/cli/notifiers/mail +24 -0
  103. data/templates/cli/notifiers/nagios +13 -0
  104. data/templates/cli/notifiers/pagerduty +12 -0
  105. data/templates/cli/notifiers/prowl +11 -0
  106. data/templates/cli/notifiers/pushover +11 -0
  107. data/templates/cli/notifiers/ses +15 -0
  108. data/templates/cli/notifiers/slack +22 -0
  109. data/templates/cli/notifiers/twitter +13 -0
  110. data/templates/cli/splitter +7 -0
  111. data/templates/cli/storages/cloud_files +11 -0
  112. data/templates/cli/storages/dropbox +20 -0
  113. data/templates/cli/storages/ftp +13 -0
  114. data/templates/cli/storages/local +8 -0
  115. data/templates/cli/storages/qiniu +12 -0
  116. data/templates/cli/storages/rsync +17 -0
  117. data/templates/cli/storages/s3 +16 -0
  118. data/templates/cli/storages/scp +15 -0
  119. data/templates/cli/storages/sftp +15 -0
  120. data/templates/cli/syncers/cloud_files +22 -0
  121. data/templates/cli/syncers/rsync_local +20 -0
  122. data/templates/cli/syncers/rsync_pull +28 -0
  123. data/templates/cli/syncers/rsync_push +28 -0
  124. data/templates/cli/syncers/s3 +27 -0
  125. data/templates/general/links +3 -0
  126. data/templates/general/version.erb +2 -0
  127. data/templates/notifier/mail/failure.erb +16 -0
  128. data/templates/notifier/mail/success.erb +16 -0
  129. data/templates/notifier/mail/warning.erb +16 -0
  130. data/templates/storage/dropbox/authorization_url.erb +6 -0
  131. data/templates/storage/dropbox/authorized.erb +4 -0
  132. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  133. metadata +1077 -0
@@ -0,0 +1,118 @@
1
+ # encoding: utf-8
2
+ require 'hipchat'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class Hipchat < Base
7
+
8
+ ##
9
+ # The Hipchat API token
10
+ attr_accessor :token
11
+
12
+ ##
13
+ # The Hipchat API version
14
+ # Either 'v1' or 'v2' (default is 'v1')
15
+ attr_accessor :api_version
16
+
17
+ ##
18
+ # Who the notification should appear from
19
+ attr_accessor :from
20
+
21
+ ##
22
+ # Custom server URL
23
+ attr_accessor :server_url
24
+
25
+ ##
26
+ # The rooms that should be notified
27
+ attr_accessor :rooms_notified
28
+
29
+ ##
30
+ # Notify users in the room
31
+ attr_accessor :notify_users
32
+
33
+ ##
34
+ # The background color of a success message.
35
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
36
+ attr_accessor :success_color
37
+
38
+ ##
39
+ # The background color of a warning message.
40
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
41
+ attr_accessor :warning_color
42
+
43
+ ##
44
+ # The background color of an error message.
45
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
46
+ attr_accessor :failure_color
47
+
48
+ def initialize(model, &block)
49
+ super
50
+ instance_eval(&block) if block_given?
51
+
52
+ @notify_users ||= false
53
+ @rooms_notified ||= []
54
+ @success_color ||= 'yellow'
55
+ @warning_color ||= 'yellow'
56
+ @failure_color ||= 'yellow'
57
+ @api_version ||= 'v1'
58
+ end
59
+
60
+ private
61
+
62
+ ##
63
+ # Notify the user of the backup operation results.
64
+ #
65
+ # `status` indicates one of the following:
66
+ #
67
+ # `:success`
68
+ # : The backup completed successfully.
69
+ # : Notification will be sent if `on_success` is `true`.
70
+ #
71
+ # `:warning`
72
+ # : The backup completed successfully, but warnings were logged.
73
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
74
+ #
75
+ # `:failure`
76
+ # : The backup operation failed.
77
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
78
+ #
79
+ def notify!(status)
80
+ status_data = status_data_for(status)
81
+ msg = message.call(model, :status => status_data)
82
+ send_message(msg, status_data[:color])
83
+ end
84
+
85
+ def client_options
86
+ { api_version: @api_version }.tap do |h|
87
+ h[:server_url] = server_url if server_url
88
+ end
89
+ end
90
+
91
+ # Hipchat::Client will raise an error if unsuccessful.
92
+ def send_message(msg, color)
93
+ client = HipChat::Client.new(token, client_options)
94
+ rooms_to_notify.each do |room|
95
+ client[room].send(from, msg, :color => color, :notify => notify_users)
96
+ end
97
+ end
98
+
99
+ def rooms_to_notify
100
+ Array(rooms_notified).map {|r| r.split(',').map(&:strip) }.flatten
101
+ end
102
+
103
+ def status_data_for(status)
104
+ data = super(status)
105
+ data[:color] = status_color_for(status)
106
+ data
107
+ end
108
+
109
+ def status_color_for(status)
110
+ {
111
+ :success => success_color,
112
+ :warning => warning_color,
113
+ :failure => failure_color
114
+ }[status]
115
+ end
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,117 @@
1
+ # encoding: utf-8
2
+ require 'uri'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class HttpPost < Base
7
+
8
+ ##
9
+ # URI to post notification to.
10
+ #
11
+ # URI scheme may be `http` or `https`.
12
+ #
13
+ # If Basic Authentication is needed, supply the `user:password` in the URI.
14
+ # e.g. 'https://user:pass@www.example.com/path'
15
+ #
16
+ # Port may also be supplied.
17
+ # e.g. 'http://www.example.com:8080/path'
18
+ attr_accessor :uri
19
+
20
+ ##
21
+ # Hash of additional HTTP headers to send.
22
+ #
23
+ # This notifier sets the following headers:
24
+ # { 'User-Agent' => "Backup/#{ Backup::VERSION }",
25
+ # 'Content-Type' => 'x-www-form-urlencoded' }
26
+ #
27
+ # 'Content-Type' may not be changed.
28
+ # 'User-Agent' may be overridden or omitted by setting it to +nil+.
29
+ # e.g. { 'Authorization' => 'my_auth_info', 'User-Agent' => nil }
30
+ attr_accessor :headers
31
+
32
+ ##
33
+ # Hash of additional POST parameters to send.
34
+ #
35
+ # This notifier will set two parameters:
36
+ # { 'status' => 'success|warning|failure',
37
+ # 'message' => '[Backup::(Success|Warning|Failure)] label (trigger)' }
38
+ #
39
+ # 'status' may not be changed.
40
+ # 'message' may be overridden or omitted by setting a +nil+ value.
41
+ # e.g. { 'auth_token' => 'my_token', 'message' => nil }
42
+ attr_accessor :params
43
+
44
+ ##
45
+ # Successful HTTP Status Code(s) that should be returned.
46
+ #
47
+ # This may be a single code or an Array of acceptable codes.
48
+ # e.g. [200, 201, 204]
49
+ #
50
+ # If any other response code is returned, the request will be retried
51
+ # using `max_retries` and `retry_waitsec`.
52
+ #
53
+ # Default: 200
54
+ attr_accessor :success_codes
55
+
56
+ ##
57
+ # Verify the server's certificate when using SSL.
58
+ #
59
+ # This will default to +true+ for most systems.
60
+ # It may be forced by setting to +true+, or disabled by setting to +false+.
61
+ attr_accessor :ssl_verify_peer
62
+
63
+ ##
64
+ # Path to a +cacert.pem+ file to use for +ssl_verify_peer+.
65
+ #
66
+ # This is provided (via Excon), but may be specified if needed.
67
+ attr_accessor :ssl_ca_file
68
+
69
+ def initialize(model, &block)
70
+ super
71
+ instance_eval(&block) if block_given?
72
+
73
+ @headers ||= {}
74
+ @params ||= {}
75
+ @success_codes ||= 200
76
+ end
77
+
78
+ private
79
+
80
+ ##
81
+ # Notify the user of the backup operation results.
82
+ #
83
+ # `status` indicates one of the following:
84
+ #
85
+ # `:success`
86
+ # : The backup completed successfully.
87
+ # : Notification will be sent if `on_success` is `true`.
88
+ #
89
+ # `:warning`
90
+ # : The backup completed successfully, but warnings were logged.
91
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
92
+ #
93
+ # `:failure`
94
+ # : The backup operation failed.
95
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
96
+ #
97
+ def notify!(status)
98
+ msg = message.call(model, :status => status_data_for(status))
99
+
100
+ opts = {
101
+ :headers => { 'User-Agent' => "Backup/#{ VERSION }" }.
102
+ merge(headers).reject {|k,v| v.nil? }.
103
+ merge('Content-Type' => 'application/x-www-form-urlencoded'),
104
+ :body => URI.encode_www_form({ 'message' => msg }.
105
+ merge(params).reject {|k,v| v.nil? }.
106
+ merge('status' => status.to_s)),
107
+ :expects => success_codes # raise error if unsuccessful
108
+ }
109
+ opts.merge!(:ssl_verify_peer => ssl_verify_peer) unless ssl_verify_peer.nil?
110
+ opts.merge!(:ssl_ca_file => ssl_ca_file) if ssl_ca_file
111
+
112
+ Excon.post(uri, opts)
113
+ end
114
+
115
+ end
116
+ end
117
+ end
@@ -0,0 +1,249 @@
1
+ # encoding: utf-8
2
+ require 'mail'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class Mail < Base
7
+
8
+ ##
9
+ # Mail delivery method to be used by the Mail gem.
10
+ #
11
+ # Supported methods:
12
+ #
13
+ # [:smtp - ::Mail::SMTP (default)]
14
+ # Settings used by this method:
15
+ # {#address}, {#port}, {#domain}, {#user_name}, {#password},
16
+ # {#authentication}, {#encryption}, {#openssl_verify_mode}
17
+ #
18
+ # [:sendmail - ::Mail::Sendmail]
19
+ # Settings used by this method:
20
+ # {#sendmail_args}
21
+ #
22
+ # [:exim - ::Mail::Exim]
23
+ # Settings used by this method:
24
+ # {#exim_args}
25
+ #
26
+ # [:file - ::Mail::FileDelivery]
27
+ # Settings used by this method:
28
+ # {#mail_folder}
29
+ #
30
+ attr_accessor :delivery_method
31
+
32
+ ##
33
+ # Sender Email Address
34
+ attr_accessor :from
35
+
36
+ ##
37
+ # Receiver Email Address
38
+ attr_accessor :to
39
+
40
+ ##
41
+ # CC receiver Email Address
42
+ attr_accessor :cc
43
+
44
+ ##
45
+ # BCC receiver Email Address
46
+ attr_accessor :bcc
47
+
48
+ ##
49
+ # Set reply to email address
50
+ attr_accessor :reply_to
51
+
52
+ ##
53
+ # SMTP Server Address
54
+ attr_accessor :address
55
+
56
+ ##
57
+ # SMTP Server Port
58
+ attr_accessor :port
59
+
60
+ ##
61
+ # Your domain (if applicable)
62
+ attr_accessor :domain
63
+
64
+ ##
65
+ # SMTP Server Username (sender email's credentials)
66
+ attr_accessor :user_name
67
+
68
+ ##
69
+ # SMTP Server Password (sender email's credentials)
70
+ attr_accessor :password
71
+
72
+ ##
73
+ # Authentication type
74
+ #
75
+ # Acceptable values: +:plain+, +:login+, +:cram_md5+
76
+ attr_accessor :authentication
77
+
78
+ ##
79
+ # Set the method of encryption to be used for the +SMTP+ connection.
80
+ #
81
+ # [:starttls (default)]
82
+ # Use +STARTTLS+ to upgrade the connection to a +SSL/TLS+ connection.
83
+ #
84
+ # [:tls or :ssl]
85
+ # Use a +SSL/TLS+ connection.
86
+ #
87
+ # [:none]
88
+ # No encryption will be used.
89
+ attr_accessor :encryption
90
+
91
+ ##
92
+ # OpenSSL Verify Mode
93
+ #
94
+ # Valid modes: +:none+, +:peer+, +:client_once+, +:fail_if_no_peer_cert+
95
+ # See +OpenSSL::SSL+ for details.
96
+ #
97
+ # Use +:none+ for a self-signed and/or wildcard certificate
98
+ attr_accessor :openssl_verify_mode
99
+
100
+ ##
101
+ # Optional arguments to pass to `sendmail`
102
+ #
103
+ # Note that this will override the defaults set by the Mail gem
104
+ # (currently: '-i'). So, if set here, be sure to set all the arguments
105
+ # you require.
106
+ #
107
+ # Example: '-i -X/tmp/traffic.log'
108
+ attr_accessor :sendmail_args
109
+
110
+ ##
111
+ # Optional arguments to pass to `exim`
112
+ #
113
+ # Note that this will override the defaults set by the Mail gem
114
+ # (currently: '-i -t') So, if set here, be sure to set all the arguments
115
+ # you require.
116
+ #
117
+ # Example: '-i -t -X/tmp/traffic.log'
118
+ attr_accessor :exim_args
119
+
120
+ ##
121
+ # Folder where mail will be kept when using the `:file` `delivery_method`.
122
+ #
123
+ # Default location is '$HOME/Backup/emails'
124
+ attr_accessor :mail_folder
125
+
126
+ ##
127
+ # Array of statuses for which the log file should be attached.
128
+ #
129
+ # Available statuses are: `:success`, `:warning` and `:failure`.
130
+ # Default: [:warning, :failure]
131
+ attr_accessor :send_log_on
132
+
133
+ def initialize(model, &block)
134
+ super
135
+ instance_eval(&block) if block_given?
136
+
137
+ @send_log_on ||= [:warning, :failure]
138
+ @encryption ||= :starttls
139
+ end
140
+
141
+ private
142
+
143
+ ##
144
+ # Notify the user of the backup operation results.
145
+ #
146
+ # `status` indicates one of the following:
147
+ #
148
+ # `:success`
149
+ # : The backup completed successfully.
150
+ # : Notification will be sent if `on_success` is `true`.
151
+ #
152
+ # `:warning`
153
+ # : The backup completed successfully, but warnings were logged.
154
+ # : Notification will be sent, including a copy of the current
155
+ # : backup log, if `on_warning` or `on_success` is `true`.
156
+ #
157
+ # `:failure`
158
+ # : The backup operation failed.
159
+ # : Notification will be sent, including a copy of the current
160
+ # : backup log, if `on_failure` is `true`.
161
+ #
162
+ def notify!(status)
163
+ email = new_email
164
+ email.subject = message.call(model, :status => status_data_for(status))
165
+
166
+ send_log = send_log_on.include?(status)
167
+ template = Backup::Template.new({ :model => model, :send_log => send_log })
168
+ email.body = template.result('notifier/mail/%s.erb' % status.to_s)
169
+
170
+ if send_log
171
+ email.convert_to_multipart
172
+ email.attachments["#{ model.time }.#{ model.trigger }.log"] = {
173
+ :mime_type => 'text/plain;',
174
+ :content => Logger.messages.map(&:formatted_lines).flatten.join("\n")
175
+ }
176
+ end
177
+
178
+ email.deliver! # raise error if unsuccessful
179
+ end
180
+
181
+ ##
182
+ # Configures the Mail gem by setting the defaults.
183
+ # Creates and returns a new email, based on the @delivery_method used.
184
+ def new_email
185
+ method = %w{ smtp sendmail exim file test }.
186
+ index(@delivery_method.to_s) ? @delivery_method.to_s : 'smtp'
187
+
188
+ options =
189
+ case method
190
+ when 'smtp'
191
+ opts = {
192
+ :address => @address,
193
+ :port => @port,
194
+ :user_name => @user_name,
195
+ :password => @password,
196
+ :authentication => @authentication,
197
+ :enable_starttls_auto => @encryption == :starttls,
198
+ :openssl_verify_mode => @openssl_verify_mode,
199
+ :ssl => @encryption == :ssl,
200
+ :tls => @encryption == :tls
201
+ }
202
+
203
+ # Don't override default domain setting if domain not applicable.
204
+ # ref https://github.com/mikel/mail/blob/2.6.3/lib/mail/network/delivery_methods/smtp.rb#L82
205
+ opts[:domain] = @domain if @domain
206
+ opts
207
+ when 'sendmail'
208
+ opts = {}
209
+ opts.merge!(:location => utility(:sendmail))
210
+ opts.merge!(:arguments => @sendmail_args) if @sendmail_args
211
+ opts
212
+ when 'exim'
213
+ opts = {}
214
+ opts.merge!(:location => utility(:exim))
215
+ opts.merge!(:arguments => @exim_args) if @exim_args
216
+ opts
217
+ when 'file'
218
+ @mail_folder ||= File.join(Config.root_path, 'emails')
219
+ { :location => File.expand_path(@mail_folder) }
220
+ when 'test' then {}
221
+ end
222
+
223
+ email = ::Mail.new
224
+ email.delivery_method method.to_sym, options
225
+ email.to = to
226
+ email.from = from
227
+ email.cc = cc
228
+ email.bcc = bcc
229
+ email.reply_to = reply_to
230
+ email
231
+ end
232
+
233
+ end
234
+ end
235
+ end
236
+
237
+ # Patch mail v2.5.4 Exim delivery method
238
+ # https://github.com/backup/backup/issues/446
239
+ # https://github.com/mikel/mail/pull/546
240
+ module Mail
241
+ class Exim
242
+ def self.call(path, arguments, destinations, encoded_message)
243
+ popen "#{path} #{arguments}" do |io|
244
+ io.puts encoded_message.to_lf
245
+ io.flush
246
+ end
247
+ end
248
+ end
249
+ end