ext_backup 5.0.0.beta.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +19 -0
  3. data/README.md +33 -0
  4. data/bin/backup +5 -0
  5. data/bin/docker_test +24 -0
  6. data/lib/backup.rb +140 -0
  7. data/lib/backup/archive.rb +169 -0
  8. data/lib/backup/binder.rb +18 -0
  9. data/lib/backup/cleaner.rb +112 -0
  10. data/lib/backup/cli.rb +370 -0
  11. data/lib/backup/cloud_io/base.rb +38 -0
  12. data/lib/backup/cloud_io/cloud_files.rb +296 -0
  13. data/lib/backup/cloud_io/s3.rb +253 -0
  14. data/lib/backup/compressor/base.rb +32 -0
  15. data/lib/backup/compressor/bzip2.rb +35 -0
  16. data/lib/backup/compressor/custom.rb +49 -0
  17. data/lib/backup/compressor/gzip.rb +73 -0
  18. data/lib/backup/config.rb +128 -0
  19. data/lib/backup/config/dsl.rb +102 -0
  20. data/lib/backup/config/helpers.rb +137 -0
  21. data/lib/backup/database/base.rb +86 -0
  22. data/lib/backup/database/mongodb.rb +186 -0
  23. data/lib/backup/database/mysql.rb +191 -0
  24. data/lib/backup/database/openldap.rb +93 -0
  25. data/lib/backup/database/postgresql.rb +132 -0
  26. data/lib/backup/database/redis.rb +176 -0
  27. data/lib/backup/database/riak.rb +79 -0
  28. data/lib/backup/database/sqlite.rb +55 -0
  29. data/lib/backup/encryptor/base.rb +27 -0
  30. data/lib/backup/encryptor/gpg.rb +737 -0
  31. data/lib/backup/encryptor/open_ssl.rb +74 -0
  32. data/lib/backup/errors.rb +53 -0
  33. data/lib/backup/logger.rb +197 -0
  34. data/lib/backup/logger/console.rb +48 -0
  35. data/lib/backup/logger/fog_adapter.rb +25 -0
  36. data/lib/backup/logger/logfile.rb +131 -0
  37. data/lib/backup/logger/syslog.rb +114 -0
  38. data/lib/backup/model.rb +472 -0
  39. data/lib/backup/notifier/base.rb +126 -0
  40. data/lib/backup/notifier/campfire.rb +61 -0
  41. data/lib/backup/notifier/command.rb +99 -0
  42. data/lib/backup/notifier/datadog.rb +104 -0
  43. data/lib/backup/notifier/flowdock.rb +99 -0
  44. data/lib/backup/notifier/hipchat.rb +116 -0
  45. data/lib/backup/notifier/http_post.rb +114 -0
  46. data/lib/backup/notifier/mail.rb +232 -0
  47. data/lib/backup/notifier/nagios.rb +65 -0
  48. data/lib/backup/notifier/pagerduty.rb +79 -0
  49. data/lib/backup/notifier/prowl.rb +68 -0
  50. data/lib/backup/notifier/pushover.rb +71 -0
  51. data/lib/backup/notifier/ses.rb +123 -0
  52. data/lib/backup/notifier/slack.rb +147 -0
  53. data/lib/backup/notifier/twitter.rb +55 -0
  54. data/lib/backup/notifier/zabbix.rb +60 -0
  55. data/lib/backup/package.rb +51 -0
  56. data/lib/backup/packager.rb +106 -0
  57. data/lib/backup/pipeline.rb +120 -0
  58. data/lib/backup/splitter.rb +73 -0
  59. data/lib/backup/storage/base.rb +66 -0
  60. data/lib/backup/storage/cloud_files.rb +156 -0
  61. data/lib/backup/storage/cycler.rb +70 -0
  62. data/lib/backup/storage/dropbox.rb +206 -0
  63. data/lib/backup/storage/ftp.rb +116 -0
  64. data/lib/backup/storage/local.rb +61 -0
  65. data/lib/backup/storage/qiniu.rb +65 -0
  66. data/lib/backup/storage/rsync.rb +246 -0
  67. data/lib/backup/storage/s3.rb +155 -0
  68. data/lib/backup/storage/scp.rb +65 -0
  69. data/lib/backup/storage/sftp.rb +80 -0
  70. data/lib/backup/syncer/base.rb +67 -0
  71. data/lib/backup/syncer/cloud/base.rb +176 -0
  72. data/lib/backup/syncer/cloud/cloud_files.rb +81 -0
  73. data/lib/backup/syncer/cloud/local_file.rb +97 -0
  74. data/lib/backup/syncer/cloud/s3.rb +109 -0
  75. data/lib/backup/syncer/rsync/base.rb +50 -0
  76. data/lib/backup/syncer/rsync/local.rb +27 -0
  77. data/lib/backup/syncer/rsync/pull.rb +47 -0
  78. data/lib/backup/syncer/rsync/push.rb +201 -0
  79. data/lib/backup/template.rb +41 -0
  80. data/lib/backup/utilities.rb +233 -0
  81. data/lib/backup/version.rb +3 -0
  82. data/lib/ext_backup.rb +5 -0
  83. data/lib/ext_backup/version.rb +5 -0
  84. data/templates/cli/archive +28 -0
  85. data/templates/cli/compressor/bzip2 +4 -0
  86. data/templates/cli/compressor/custom +7 -0
  87. data/templates/cli/compressor/gzip +4 -0
  88. data/templates/cli/config +123 -0
  89. data/templates/cli/databases/mongodb +15 -0
  90. data/templates/cli/databases/mysql +18 -0
  91. data/templates/cli/databases/openldap +24 -0
  92. data/templates/cli/databases/postgresql +16 -0
  93. data/templates/cli/databases/redis +16 -0
  94. data/templates/cli/databases/riak +17 -0
  95. data/templates/cli/databases/sqlite +11 -0
  96. data/templates/cli/encryptor/gpg +27 -0
  97. data/templates/cli/encryptor/openssl +9 -0
  98. data/templates/cli/model +26 -0
  99. data/templates/cli/notifier/zabbix +15 -0
  100. data/templates/cli/notifiers/campfire +12 -0
  101. data/templates/cli/notifiers/command +32 -0
  102. data/templates/cli/notifiers/datadog +57 -0
  103. data/templates/cli/notifiers/flowdock +16 -0
  104. data/templates/cli/notifiers/hipchat +16 -0
  105. data/templates/cli/notifiers/http_post +32 -0
  106. data/templates/cli/notifiers/mail +24 -0
  107. data/templates/cli/notifiers/nagios +13 -0
  108. data/templates/cli/notifiers/pagerduty +12 -0
  109. data/templates/cli/notifiers/prowl +11 -0
  110. data/templates/cli/notifiers/pushover +11 -0
  111. data/templates/cli/notifiers/ses +15 -0
  112. data/templates/cli/notifiers/slack +22 -0
  113. data/templates/cli/notifiers/twitter +13 -0
  114. data/templates/cli/splitter +7 -0
  115. data/templates/cli/storages/cloud_files +11 -0
  116. data/templates/cli/storages/dropbox +20 -0
  117. data/templates/cli/storages/ftp +13 -0
  118. data/templates/cli/storages/local +8 -0
  119. data/templates/cli/storages/qiniu +12 -0
  120. data/templates/cli/storages/rsync +17 -0
  121. data/templates/cli/storages/s3 +16 -0
  122. data/templates/cli/storages/scp +15 -0
  123. data/templates/cli/storages/sftp +15 -0
  124. data/templates/cli/syncers/cloud_files +22 -0
  125. data/templates/cli/syncers/rsync_local +20 -0
  126. data/templates/cli/syncers/rsync_pull +28 -0
  127. data/templates/cli/syncers/rsync_push +28 -0
  128. data/templates/cli/syncers/s3 +27 -0
  129. data/templates/general/links +3 -0
  130. data/templates/general/version.erb +2 -0
  131. data/templates/notifier/mail/failure.erb +16 -0
  132. data/templates/notifier/mail/success.erb +16 -0
  133. data/templates/notifier/mail/warning.erb +16 -0
  134. data/templates/storage/dropbox/authorization_url.erb +6 -0
  135. data/templates/storage/dropbox/authorized.erb +4 -0
  136. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  137. metadata +506 -0
@@ -0,0 +1,116 @@
1
+ require "hipchat"
2
+
3
+ module Backup
4
+ module Notifier
5
+ class Hipchat < Base
6
+ ##
7
+ # The Hipchat API token
8
+ attr_accessor :token
9
+
10
+ ##
11
+ # The Hipchat API version
12
+ # Either 'v1' or 'v2' (default is 'v1')
13
+ attr_accessor :api_version
14
+
15
+ ##
16
+ # Who the notification should appear from
17
+ attr_accessor :from
18
+
19
+ ##
20
+ # Custom server URL
21
+ attr_accessor :server_url
22
+
23
+ ##
24
+ # The rooms that should be notified
25
+ attr_accessor :rooms_notified
26
+
27
+ ##
28
+ # Notify users in the room
29
+ attr_accessor :notify_users
30
+
31
+ ##
32
+ # The background color of a success message.
33
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
34
+ attr_accessor :success_color
35
+
36
+ ##
37
+ # The background color of a warning message.
38
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
39
+ attr_accessor :warning_color
40
+
41
+ ##
42
+ # The background color of an error message.
43
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
44
+ attr_accessor :failure_color
45
+
46
+ def initialize(model, &block)
47
+ super
48
+ instance_eval(&block) if block_given?
49
+
50
+ @notify_users ||= false
51
+ @rooms_notified ||= []
52
+ @success_color ||= "yellow"
53
+ @warning_color ||= "yellow"
54
+ @failure_color ||= "yellow"
55
+ @api_version ||= "v1"
56
+ end
57
+
58
+ private
59
+
60
+ ##
61
+ # Notify the user of the backup operation results.
62
+ #
63
+ # `status` indicates one of the following:
64
+ #
65
+ # `:success`
66
+ # : The backup completed successfully.
67
+ # : Notification will be sent if `on_success` is `true`.
68
+ #
69
+ # `:warning`
70
+ # : The backup completed successfully, but warnings were logged.
71
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
72
+ #
73
+ # `:failure`
74
+ # : The backup operation failed.
75
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
76
+ #
77
+ def notify!(status)
78
+ status_data = status_data_for(status)
79
+ msg = message.call(model, status: status_data)
80
+ send_message(msg, status_data[:color])
81
+ end
82
+
83
+ def client_options
84
+ { api_version: @api_version }.tap do |h|
85
+ h[:server_url] = server_url if server_url
86
+ end
87
+ end
88
+
89
+ # Hipchat::Client will raise an error if unsuccessful.
90
+ def send_message(msg, color)
91
+ client = HipChat::Client.new(token, client_options)
92
+ rooms_to_notify.each do |room|
93
+ client[room].send(from, msg, color: color, notify: notify_users)
94
+ end
95
+ end
96
+
97
+ def rooms_to_notify
98
+ Array(rooms_notified).map { |r| r.split(",").map(&:strip) }.flatten
99
+ end
100
+
101
+ def status_data_for(status)
102
+ data = super(status)
103
+ data[:color] = status_color_for(status)
104
+ data
105
+ end
106
+
107
+ def status_color_for(status)
108
+ {
109
+ success: success_color,
110
+ warning: warning_color,
111
+ failure: failure_color
112
+ }[status]
113
+ end
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,114 @@
1
+ require "uri"
2
+
3
+ module Backup
4
+ module Notifier
5
+ class HttpPost < Base
6
+ ##
7
+ # URI to post notification to.
8
+ #
9
+ # URI scheme may be `http` or `https`.
10
+ #
11
+ # If Basic Authentication is needed, supply the `user:password` in the URI.
12
+ # e.g. 'https://user:pass@www.example.com/path'
13
+ #
14
+ # Port may also be supplied.
15
+ # e.g. 'http://www.example.com:8080/path'
16
+ attr_accessor :uri
17
+
18
+ ##
19
+ # Hash of additional HTTP headers to send.
20
+ #
21
+ # This notifier sets the following headers:
22
+ # { 'User-Agent' => "Backup/#{ Backup::VERSION }",
23
+ # 'Content-Type' => 'x-www-form-urlencoded' }
24
+ #
25
+ # 'Content-Type' may not be changed.
26
+ # 'User-Agent' may be overridden or omitted by setting it to +nil+.
27
+ # e.g. { 'Authorization' => 'my_auth_info', 'User-Agent' => nil }
28
+ attr_accessor :headers
29
+
30
+ ##
31
+ # Hash of additional POST parameters to send.
32
+ #
33
+ # This notifier will set two parameters:
34
+ # { 'status' => 'success|warning|failure',
35
+ # 'message' => '[Backup::(Success|Warning|Failure)] label (trigger)' }
36
+ #
37
+ # 'status' may not be changed.
38
+ # 'message' may be overridden or omitted by setting a +nil+ value.
39
+ # e.g. { 'auth_token' => 'my_token', 'message' => nil }
40
+ attr_accessor :params
41
+
42
+ ##
43
+ # Successful HTTP Status Code(s) that should be returned.
44
+ #
45
+ # This may be a single code or an Array of acceptable codes.
46
+ # e.g. [200, 201, 204]
47
+ #
48
+ # If any other response code is returned, the request will be retried
49
+ # using `max_retries` and `retry_waitsec`.
50
+ #
51
+ # Default: 200
52
+ attr_accessor :success_codes
53
+
54
+ ##
55
+ # Verify the server's certificate when using SSL.
56
+ #
57
+ # This will default to +true+ for most systems.
58
+ # It may be forced by setting to +true+, or disabled by setting to +false+.
59
+ attr_accessor :ssl_verify_peer
60
+
61
+ ##
62
+ # Path to a +cacert.pem+ file to use for +ssl_verify_peer+.
63
+ #
64
+ # This is provided (via Excon), but may be specified if needed.
65
+ attr_accessor :ssl_ca_file
66
+
67
+ def initialize(model, &block)
68
+ super
69
+ instance_eval(&block) if block_given?
70
+
71
+ @headers ||= {}
72
+ @params ||= {}
73
+ @success_codes ||= 200
74
+ end
75
+
76
+ private
77
+
78
+ ##
79
+ # Notify the user of the backup operation results.
80
+ #
81
+ # `status` indicates one of the following:
82
+ #
83
+ # `:success`
84
+ # : The backup completed successfully.
85
+ # : Notification will be sent if `on_success` is `true`.
86
+ #
87
+ # `:warning`
88
+ # : The backup completed successfully, but warnings were logged.
89
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
90
+ #
91
+ # `:failure`
92
+ # : The backup operation failed.
93
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
94
+ #
95
+ def notify!(status)
96
+ msg = message.call(model, status: status_data_for(status))
97
+
98
+ opts = {
99
+ headers: { "User-Agent" => "Backup/#{VERSION}" }
100
+ .merge(headers).reject { |_, value| value.nil? }
101
+ .merge("Content-Type" => "application/x-www-form-urlencoded"),
102
+ body: URI.encode_www_form({ "message" => msg }
103
+ .merge(params).reject { |_, value| value.nil? }
104
+ .merge("status" => status.to_s)),
105
+ expects: success_codes # raise error if unsuccessful
106
+ }
107
+ opts[:ssl_verify_peer] = ssl_verify_peer unless ssl_verify_peer.nil?
108
+ opts[:ssl_ca_file] = ssl_ca_file if ssl_ca_file
109
+
110
+ Excon.post(uri, opts)
111
+ end
112
+ end
113
+ end
114
+ end
@@ -0,0 +1,232 @@
1
+ require "mail"
2
+
3
+ module Backup
4
+ module Notifier
5
+ class Mail < Base
6
+ ##
7
+ # Mail delivery method to be used by the Mail gem.
8
+ #
9
+ # Supported methods:
10
+ #
11
+ # [:smtp - ::Mail::SMTP (default)]
12
+ # Settings used by this method:
13
+ # {#address}, {#port}, {#domain}, {#user_name}, {#password},
14
+ # {#authentication}, {#encryption}, {#openssl_verify_mode}
15
+ #
16
+ # [:sendmail - ::Mail::Sendmail]
17
+ # Settings used by this method:
18
+ # {#sendmail_args}
19
+ #
20
+ # [:exim - ::Mail::Exim]
21
+ # Settings used by this method:
22
+ # {#exim_args}
23
+ #
24
+ # [:file - ::Mail::FileDelivery]
25
+ # Settings used by this method:
26
+ # {#mail_folder}
27
+ #
28
+ attr_accessor :delivery_method
29
+
30
+ ##
31
+ # Sender Email Address
32
+ attr_accessor :from
33
+
34
+ ##
35
+ # Receiver Email Address
36
+ attr_accessor :to
37
+
38
+ ##
39
+ # CC receiver Email Address
40
+ attr_accessor :cc
41
+
42
+ ##
43
+ # BCC receiver Email Address
44
+ attr_accessor :bcc
45
+
46
+ ##
47
+ # Set reply to email address
48
+ attr_accessor :reply_to
49
+
50
+ ##
51
+ # SMTP Server Address
52
+ attr_accessor :address
53
+
54
+ ##
55
+ # SMTP Server Port
56
+ attr_accessor :port
57
+
58
+ ##
59
+ # Your domain (if applicable)
60
+ attr_accessor :domain
61
+
62
+ ##
63
+ # SMTP Server Username (sender email's credentials)
64
+ attr_accessor :user_name
65
+
66
+ ##
67
+ # SMTP Server Password (sender email's credentials)
68
+ attr_accessor :password
69
+
70
+ ##
71
+ # Authentication type
72
+ #
73
+ # Acceptable values: +:plain+, +:login+, +:cram_md5+
74
+ attr_accessor :authentication
75
+
76
+ ##
77
+ # Set the method of encryption to be used for the +SMTP+ connection.
78
+ #
79
+ # [:starttls (default)]
80
+ # Use +STARTTLS+ to upgrade the connection to a +SSL/TLS+ connection.
81
+ #
82
+ # [:tls or :ssl]
83
+ # Use a +SSL/TLS+ connection.
84
+ #
85
+ # [:none]
86
+ # No encryption will be used.
87
+ attr_accessor :encryption
88
+
89
+ ##
90
+ # OpenSSL Verify Mode
91
+ #
92
+ # Valid modes: +:none+, +:peer+, +:client_once+, +:fail_if_no_peer_cert+
93
+ # See +OpenSSL::SSL+ for details.
94
+ #
95
+ # Use +:none+ for a self-signed and/or wildcard certificate
96
+ attr_accessor :openssl_verify_mode
97
+
98
+ ##
99
+ # Optional arguments to pass to `sendmail`
100
+ #
101
+ # Note that this will override the defaults set by the Mail gem
102
+ # (currently: '-i'). So, if set here, be sure to set all the arguments
103
+ # you require.
104
+ #
105
+ # Example: '-i -X/tmp/traffic.log'
106
+ attr_accessor :sendmail_args
107
+
108
+ ##
109
+ # Optional arguments to pass to `exim`
110
+ #
111
+ # Note that this will override the defaults set by the Mail gem
112
+ # (currently: '-i -t') So, if set here, be sure to set all the arguments
113
+ # you require.
114
+ #
115
+ # Example: '-i -t -X/tmp/traffic.log'
116
+ attr_accessor :exim_args
117
+
118
+ ##
119
+ # Folder where mail will be kept when using the `:file` `delivery_method`.
120
+ #
121
+ # Default location is '$HOME/Backup/emails'
122
+ attr_accessor :mail_folder
123
+
124
+ ##
125
+ # Array of statuses for which the log file should be attached.
126
+ #
127
+ # Available statuses are: `:success`, `:warning` and `:failure`.
128
+ # Default: [:warning, :failure]
129
+ attr_accessor :send_log_on
130
+
131
+ def initialize(model, &block)
132
+ super
133
+ instance_eval(&block) if block_given?
134
+
135
+ @send_log_on ||= [:warning, :failure]
136
+ @encryption ||= :starttls
137
+ end
138
+
139
+ private
140
+
141
+ ##
142
+ # Notify the user of the backup operation results.
143
+ #
144
+ # `status` indicates one of the following:
145
+ #
146
+ # `:success`
147
+ # : The backup completed successfully.
148
+ # : Notification will be sent if `on_success` is `true`.
149
+ #
150
+ # `:warning`
151
+ # : The backup completed successfully, but warnings were logged.
152
+ # : Notification will be sent, including a copy of the current
153
+ # : backup log, if `on_warning` or `on_success` is `true`.
154
+ #
155
+ # `:failure`
156
+ # : The backup operation failed.
157
+ # : Notification will be sent, including a copy of the current
158
+ # : backup log, if `on_failure` is `true`.
159
+ #
160
+ def notify!(status)
161
+ email = new_email
162
+ email.subject = message.call(model, status: status_data_for(status))
163
+
164
+ send_log = send_log_on.include?(status)
165
+ template = Backup::Template.new(model: model, send_log: send_log)
166
+ email.body = template.result(sprintf("notifier/mail/%s.erb", status.to_s))
167
+
168
+ if send_log
169
+ email.convert_to_multipart
170
+ email.attachments["#{model.time}.#{model.trigger}.log"] = {
171
+ mime_type: "text/plain;",
172
+ content: Logger.messages.map(&:formatted_lines).flatten.join("\n")
173
+ }
174
+ end
175
+
176
+ email.deliver! # raise error if unsuccessful
177
+ end
178
+
179
+ ##
180
+ # Configures the Mail gem by setting the defaults.
181
+ # Creates and returns a new email, based on the @delivery_method used.
182
+ def new_email
183
+ method = %w[smtp sendmail exim file test]
184
+ .index(@delivery_method.to_s) ? @delivery_method.to_s : "smtp"
185
+
186
+ options =
187
+ case method
188
+ when "smtp"
189
+ opts = {
190
+ address: @address,
191
+ port: @port,
192
+ user_name: @user_name,
193
+ password: @password,
194
+ authentication: @authentication,
195
+ enable_starttls_auto: @encryption == :starttls,
196
+ openssl_verify_mode: @openssl_verify_mode,
197
+ ssl: @encryption == :ssl,
198
+ tls: @encryption == :tls
199
+ }
200
+
201
+ # Don't override default domain setting if domain not applicable.
202
+ # ref https://github.com/mikel/mail/blob/2.6.3/lib/mail/network/delivery_methods/smtp.rb#L82
203
+ opts[:domain] = @domain if @domain
204
+ opts
205
+ when "sendmail"
206
+ opts = {}
207
+ opts[:location] = utility(:sendmail)
208
+ opts[:arguments] = @sendmail_args if @sendmail_args
209
+ opts
210
+ when "exim"
211
+ opts = {}
212
+ opts[:location] = utility(:exim)
213
+ opts[:arguments] = @exim_args if @exim_args
214
+ opts
215
+ when "file"
216
+ @mail_folder ||= File.join(Config.root_path, "emails")
217
+ { location: File.expand_path(@mail_folder) }
218
+ when "test" then {}
219
+ end
220
+
221
+ email = ::Mail.new
222
+ email.delivery_method method.to_sym, options
223
+ email.to = to
224
+ email.from = from
225
+ email.cc = cc
226
+ email.bcc = bcc
227
+ email.reply_to = reply_to
228
+ email
229
+ end
230
+ end
231
+ end
232
+ end