ext_backup 5.0.0.beta.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +19 -0
  3. data/README.md +33 -0
  4. data/bin/backup +5 -0
  5. data/bin/docker_test +24 -0
  6. data/lib/backup.rb +140 -0
  7. data/lib/backup/archive.rb +169 -0
  8. data/lib/backup/binder.rb +18 -0
  9. data/lib/backup/cleaner.rb +112 -0
  10. data/lib/backup/cli.rb +370 -0
  11. data/lib/backup/cloud_io/base.rb +38 -0
  12. data/lib/backup/cloud_io/cloud_files.rb +296 -0
  13. data/lib/backup/cloud_io/s3.rb +253 -0
  14. data/lib/backup/compressor/base.rb +32 -0
  15. data/lib/backup/compressor/bzip2.rb +35 -0
  16. data/lib/backup/compressor/custom.rb +49 -0
  17. data/lib/backup/compressor/gzip.rb +73 -0
  18. data/lib/backup/config.rb +128 -0
  19. data/lib/backup/config/dsl.rb +102 -0
  20. data/lib/backup/config/helpers.rb +137 -0
  21. data/lib/backup/database/base.rb +86 -0
  22. data/lib/backup/database/mongodb.rb +186 -0
  23. data/lib/backup/database/mysql.rb +191 -0
  24. data/lib/backup/database/openldap.rb +93 -0
  25. data/lib/backup/database/postgresql.rb +132 -0
  26. data/lib/backup/database/redis.rb +176 -0
  27. data/lib/backup/database/riak.rb +79 -0
  28. data/lib/backup/database/sqlite.rb +55 -0
  29. data/lib/backup/encryptor/base.rb +27 -0
  30. data/lib/backup/encryptor/gpg.rb +737 -0
  31. data/lib/backup/encryptor/open_ssl.rb +74 -0
  32. data/lib/backup/errors.rb +53 -0
  33. data/lib/backup/logger.rb +197 -0
  34. data/lib/backup/logger/console.rb +48 -0
  35. data/lib/backup/logger/fog_adapter.rb +25 -0
  36. data/lib/backup/logger/logfile.rb +131 -0
  37. data/lib/backup/logger/syslog.rb +114 -0
  38. data/lib/backup/model.rb +472 -0
  39. data/lib/backup/notifier/base.rb +126 -0
  40. data/lib/backup/notifier/campfire.rb +61 -0
  41. data/lib/backup/notifier/command.rb +99 -0
  42. data/lib/backup/notifier/datadog.rb +104 -0
  43. data/lib/backup/notifier/flowdock.rb +99 -0
  44. data/lib/backup/notifier/hipchat.rb +116 -0
  45. data/lib/backup/notifier/http_post.rb +114 -0
  46. data/lib/backup/notifier/mail.rb +232 -0
  47. data/lib/backup/notifier/nagios.rb +65 -0
  48. data/lib/backup/notifier/pagerduty.rb +79 -0
  49. data/lib/backup/notifier/prowl.rb +68 -0
  50. data/lib/backup/notifier/pushover.rb +71 -0
  51. data/lib/backup/notifier/ses.rb +123 -0
  52. data/lib/backup/notifier/slack.rb +147 -0
  53. data/lib/backup/notifier/twitter.rb +55 -0
  54. data/lib/backup/notifier/zabbix.rb +60 -0
  55. data/lib/backup/package.rb +51 -0
  56. data/lib/backup/packager.rb +106 -0
  57. data/lib/backup/pipeline.rb +120 -0
  58. data/lib/backup/splitter.rb +73 -0
  59. data/lib/backup/storage/base.rb +66 -0
  60. data/lib/backup/storage/cloud_files.rb +156 -0
  61. data/lib/backup/storage/cycler.rb +70 -0
  62. data/lib/backup/storage/dropbox.rb +206 -0
  63. data/lib/backup/storage/ftp.rb +116 -0
  64. data/lib/backup/storage/local.rb +61 -0
  65. data/lib/backup/storage/qiniu.rb +65 -0
  66. data/lib/backup/storage/rsync.rb +246 -0
  67. data/lib/backup/storage/s3.rb +155 -0
  68. data/lib/backup/storage/scp.rb +65 -0
  69. data/lib/backup/storage/sftp.rb +80 -0
  70. data/lib/backup/syncer/base.rb +67 -0
  71. data/lib/backup/syncer/cloud/base.rb +176 -0
  72. data/lib/backup/syncer/cloud/cloud_files.rb +81 -0
  73. data/lib/backup/syncer/cloud/local_file.rb +97 -0
  74. data/lib/backup/syncer/cloud/s3.rb +109 -0
  75. data/lib/backup/syncer/rsync/base.rb +50 -0
  76. data/lib/backup/syncer/rsync/local.rb +27 -0
  77. data/lib/backup/syncer/rsync/pull.rb +47 -0
  78. data/lib/backup/syncer/rsync/push.rb +201 -0
  79. data/lib/backup/template.rb +41 -0
  80. data/lib/backup/utilities.rb +233 -0
  81. data/lib/backup/version.rb +3 -0
  82. data/lib/ext_backup.rb +5 -0
  83. data/lib/ext_backup/version.rb +5 -0
  84. data/templates/cli/archive +28 -0
  85. data/templates/cli/compressor/bzip2 +4 -0
  86. data/templates/cli/compressor/custom +7 -0
  87. data/templates/cli/compressor/gzip +4 -0
  88. data/templates/cli/config +123 -0
  89. data/templates/cli/databases/mongodb +15 -0
  90. data/templates/cli/databases/mysql +18 -0
  91. data/templates/cli/databases/openldap +24 -0
  92. data/templates/cli/databases/postgresql +16 -0
  93. data/templates/cli/databases/redis +16 -0
  94. data/templates/cli/databases/riak +17 -0
  95. data/templates/cli/databases/sqlite +11 -0
  96. data/templates/cli/encryptor/gpg +27 -0
  97. data/templates/cli/encryptor/openssl +9 -0
  98. data/templates/cli/model +26 -0
  99. data/templates/cli/notifier/zabbix +15 -0
  100. data/templates/cli/notifiers/campfire +12 -0
  101. data/templates/cli/notifiers/command +32 -0
  102. data/templates/cli/notifiers/datadog +57 -0
  103. data/templates/cli/notifiers/flowdock +16 -0
  104. data/templates/cli/notifiers/hipchat +16 -0
  105. data/templates/cli/notifiers/http_post +32 -0
  106. data/templates/cli/notifiers/mail +24 -0
  107. data/templates/cli/notifiers/nagios +13 -0
  108. data/templates/cli/notifiers/pagerduty +12 -0
  109. data/templates/cli/notifiers/prowl +11 -0
  110. data/templates/cli/notifiers/pushover +11 -0
  111. data/templates/cli/notifiers/ses +15 -0
  112. data/templates/cli/notifiers/slack +22 -0
  113. data/templates/cli/notifiers/twitter +13 -0
  114. data/templates/cli/splitter +7 -0
  115. data/templates/cli/storages/cloud_files +11 -0
  116. data/templates/cli/storages/dropbox +20 -0
  117. data/templates/cli/storages/ftp +13 -0
  118. data/templates/cli/storages/local +8 -0
  119. data/templates/cli/storages/qiniu +12 -0
  120. data/templates/cli/storages/rsync +17 -0
  121. data/templates/cli/storages/s3 +16 -0
  122. data/templates/cli/storages/scp +15 -0
  123. data/templates/cli/storages/sftp +15 -0
  124. data/templates/cli/syncers/cloud_files +22 -0
  125. data/templates/cli/syncers/rsync_local +20 -0
  126. data/templates/cli/syncers/rsync_pull +28 -0
  127. data/templates/cli/syncers/rsync_push +28 -0
  128. data/templates/cli/syncers/s3 +27 -0
  129. data/templates/general/links +3 -0
  130. data/templates/general/version.erb +2 -0
  131. data/templates/notifier/mail/failure.erb +16 -0
  132. data/templates/notifier/mail/success.erb +16 -0
  133. data/templates/notifier/mail/warning.erb +16 -0
  134. data/templates/storage/dropbox/authorization_url.erb +6 -0
  135. data/templates/storage/dropbox/authorized.erb +4 -0
  136. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  137. metadata +506 -0
@@ -0,0 +1,126 @@
1
+ module Backup
2
+ module Notifier
3
+ class Error < Backup::Error; end
4
+
5
+ class Base
6
+ include Utilities::Helpers
7
+ include Config::Helpers
8
+
9
+ ##
10
+ # When set to true, the user will be notified by email
11
+ # when a backup process ends without raising any exceptions
12
+ attr_accessor :on_success
13
+ alias :notify_on_success? :on_success
14
+
15
+ ##
16
+ # When set to true, the user will be notified by email
17
+ # when a backup process is successful, but has warnings
18
+ attr_accessor :on_warning
19
+ alias :notify_on_warning? :on_warning
20
+
21
+ ##
22
+ # When set to true, the user will be notified by email
23
+ # when a backup process raises an exception before finishing
24
+ attr_accessor :on_failure
25
+ alias :notify_on_failure? :on_failure
26
+
27
+ ##
28
+ # Number of times to retry failed attempts to send notification.
29
+ # Default: 10
30
+ attr_accessor :max_retries
31
+
32
+ ##
33
+ # Time in seconds to pause before each retry.
34
+ # Default: 30
35
+ attr_accessor :retry_waitsec
36
+
37
+ ##
38
+ # Message to send. Depends on notifier implementation if this is used.
39
+ # Default: lambda returning:
40
+ # "#{ message } #{ model.label } (#{ model.trigger })"
41
+ #
42
+ # @yieldparam [model] Backup::Model
43
+ # @yieldparam [data] Hash containing `message` and `key` values.
44
+ attr_accessor :message
45
+
46
+ attr_reader :model
47
+
48
+ def initialize(model)
49
+ @model = model
50
+ load_defaults!
51
+
52
+ @on_success = true if on_success.nil?
53
+ @on_warning = true if on_warning.nil?
54
+ @on_failure = true if on_failure.nil?
55
+ @max_retries ||= 10
56
+ @retry_waitsec ||= 30
57
+ @message ||= lambda do |m, data|
58
+ "[#{data[:status][:message]}] #{m.label} (#{m.trigger})"
59
+ end
60
+ end
61
+
62
+ # This method is called from an ensure block in Model#perform! and must
63
+ # not raise any exceptions. However, each Notifier's #notify! method
64
+ # should raise an exception if the request fails so it may be retried.
65
+ def perform!
66
+ status =
67
+ case model.exit_status
68
+ when 0
69
+ :success if notify_on_success?
70
+ when 1
71
+ :warning if notify_on_success? || notify_on_warning?
72
+ else
73
+ :failure if notify_on_failure?
74
+ end
75
+
76
+ if status
77
+ Logger.info "Sending notification using #{notifier_name}..."
78
+ with_retries { notify!(status) }
79
+ end
80
+ rescue Exception => err
81
+ Logger.error Error.wrap(err, "#{notifier_name} Failed!")
82
+ end
83
+
84
+ private
85
+
86
+ def with_retries
87
+ retries = 0
88
+ begin
89
+ yield
90
+ rescue StandardError, Timeout::Error => err
91
+ retries += 1
92
+ raise if retries > max_retries
93
+
94
+ Logger.info Error.wrap(err, "Retry ##{retries} of #{max_retries}.")
95
+ sleep(retry_waitsec)
96
+ retry
97
+ end
98
+ end
99
+
100
+ ##
101
+ # Return the notifier name, with Backup namespace removed
102
+ def notifier_name
103
+ self.class.to_s.sub("Backup::", "")
104
+ end
105
+
106
+ ##
107
+ # Return status data for message creation
108
+ def status_data_for(status)
109
+ {
110
+ success: {
111
+ message: "Backup::Success",
112
+ key: :success
113
+ },
114
+ warning: {
115
+ message: "Backup::Warning",
116
+ key: :warning
117
+ },
118
+ failure: {
119
+ message: "Backup::Failure",
120
+ key: :failure
121
+ }
122
+ }[status]
123
+ end
124
+ end
125
+ end
126
+ end
@@ -0,0 +1,61 @@
1
+ require "json"
2
+
3
+ module Backup
4
+ module Notifier
5
+ class Campfire < Base
6
+ ##
7
+ # Campfire api authentication token
8
+ attr_accessor :api_token
9
+
10
+ ##
11
+ # Campfire account's subdomain
12
+ attr_accessor :subdomain
13
+
14
+ ##
15
+ # Campfire account's room id
16
+ attr_accessor :room_id
17
+
18
+ def initialize(model, &block)
19
+ super
20
+ instance_eval(&block) if block_given?
21
+ end
22
+
23
+ private
24
+
25
+ ##
26
+ # Notify the user of the backup operation results.
27
+ #
28
+ # `status` indicates one of the following:
29
+ #
30
+ # `:success`
31
+ # : The backup completed successfully.
32
+ # : Notification will be sent if `on_success` is `true`.
33
+ #
34
+ # `:warning`
35
+ # : The backup completed successfully, but warnings were logged.
36
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
37
+ #
38
+ # `:failure`
39
+ # : The backup operation failed.
40
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
41
+ #
42
+ def notify!(status)
43
+ send_message(message.call(model, status: status_data_for(status)))
44
+ end
45
+
46
+ def send_message(message)
47
+ uri = "https://#{subdomain}.campfirenow.com/room/#{room_id}/speak.json"
48
+ options = {
49
+ headers: { "Content-Type" => "application/json" },
50
+ body: JSON.dump(
51
+ message: { body: message, type: "Textmessage" }
52
+ )
53
+ }
54
+ options[:user] = api_token
55
+ options[:password] = "x" # Basic Auth
56
+ options[:expects] = 201 # raise error if unsuccessful
57
+ Excon.post(uri, options)
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,99 @@
1
+ module Backup
2
+ module Notifier
3
+ class Command < Base
4
+ ##
5
+ # Command to execute.
6
+ #
7
+ # Make sure it is accessible from your $PATH, or provide
8
+ # the absolute path to the command.
9
+ attr_accessor :command
10
+
11
+ ##
12
+ # Arguments to pass to the command.
13
+ #
14
+ # Must be an array of strings or callable objects.
15
+ #
16
+ # Callables will be invoked with #call(model, status),
17
+ # and the return value used as the argument.
18
+ #
19
+ # In strings you can use the following placeholders:
20
+ #
21
+ # %l - Model label
22
+ # %t - Model trigger
23
+ # %d - Backup duration (HH:MM:SS)
24
+ # %s - Status (success/failure/warning)
25
+ # %v - Status verb (succeeded/failed/succeeded with warnings)
26
+ #
27
+ # All placeholders can be used with uppercase letters to capitalize
28
+ # the value.
29
+ #
30
+ # Defaults to ["%L %v"]
31
+ attr_accessor :args
32
+
33
+ def initialize(model, &block)
34
+ super
35
+ instance_eval(&block) if block_given?
36
+
37
+ @args ||= ["%L %v"]
38
+ end
39
+
40
+ private
41
+
42
+ ##
43
+ # Notify the user of the backup operation results.
44
+ #
45
+ # `status` indicates one of the following:
46
+ #
47
+ # `:success`
48
+ # : The backup completed successfully.
49
+ # : Notification will be sent if `on_success` is `true`.
50
+ #
51
+ # `:warning`
52
+ # : The backup completed successfully, but warnings were logged.
53
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
54
+ #
55
+ # `:failure`
56
+ # : The backup operation failed.
57
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
58
+ #
59
+ def notify!(status)
60
+ IO.popen([@command] + args.map { |arg| format_arg(arg, status) })
61
+ end
62
+
63
+ def format_arg(arg, status)
64
+ if arg.respond_to?(:call)
65
+ arg.call(model, status)
66
+ else
67
+ arg.gsub(/%(\w)/) do |match|
68
+ ph = match[1]
69
+ val = case ph.downcase
70
+ when "l"
71
+ model.label
72
+ when "t"
73
+ model.trigger.to_s
74
+ when "d"
75
+ model.duration
76
+ when "v"
77
+ status_verb(status)
78
+ when "s"
79
+ status.to_s
80
+ end
81
+ val.capitalize! if ph == ph.upcase
82
+ val
83
+ end
84
+ end
85
+ end
86
+
87
+ def status_verb(status)
88
+ case status
89
+ when :success
90
+ "succeeded"
91
+ when :failure
92
+ "failed"
93
+ when :warning
94
+ "succeeded with warnings"
95
+ end
96
+ end
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,104 @@
1
+ require "dogapi"
2
+
3
+ module Backup
4
+ module Notifier
5
+ class DataDog < Base
6
+ ##
7
+ # The DataDog API key
8
+ attr_accessor :api_key
9
+
10
+ ##
11
+ # The title of the event
12
+ attr_accessor :title
13
+
14
+ attr_deprecate :text,
15
+ version: "4.2",
16
+ message: "Please use the `message` attribute. For more information "\
17
+ "see https://github.com/backup/backup/pull/698"
18
+
19
+ ##
20
+ # The timestamp for the event
21
+ attr_accessor :date_happened
22
+
23
+ ##
24
+ # The priority of the event (low/normal)
25
+ attr_accessor :priority
26
+
27
+ ##
28
+ # The host that generated the event
29
+ attr_accessor :host
30
+
31
+ ##
32
+ # The tags for this host (should be an array)
33
+ attr_accessor :tags
34
+
35
+ ##
36
+ # The alert_type of the event (error/warning/info/success)
37
+ attr_accessor :alert_type
38
+
39
+ ##
40
+ # The aggregation_key for the event
41
+ attr_accessor :aggregation_key
42
+
43
+ ##
44
+ # The source_type for the event (nagios, hudson, jenkins, user, my apps, feed, chef, puppet, git, bitbucket, fabric, capistrano)
45
+ attr_accessor :source_type_name
46
+
47
+ def initialize(model, &block)
48
+ super
49
+ instance_eval(&block) if block_given?
50
+ @title ||= "Backup #{model.label}"
51
+ end
52
+
53
+ private
54
+
55
+ ##
56
+ # Notify the user of the backup operation results.
57
+ #
58
+ # `status` indicates one of the following:
59
+ #
60
+ # `:success`
61
+ # : The backup completed successfully.
62
+ # : Notification will be sent if `on_success` is `true`.
63
+ #
64
+ # `:warning`
65
+ # : The backup completed successfully, but warnings were logged.
66
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
67
+ #
68
+ # `:failure`
69
+ # : The backup operation failed.
70
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
71
+ #
72
+ def notify!(status)
73
+ msg = message.call(model, status: status_data_for(status))
74
+
75
+ hash = { alert_type: default_alert_type(status) }
76
+ hash.store(:msg_title, @title)
77
+ hash.store(:date_happened, @date_happened) if @date_happened
78
+ hash.store(:priority, @priority) if @priority
79
+ hash.store(:host, @host) if @host
80
+ hash.store(:tags, @tags) if @tags
81
+ hash.store(:aggregation_key, @aggregation_key) if @aggregation_key
82
+ hash.store(:source_type_name, @source_type_name) if @source_type_name
83
+ hash.store(:alert_type, @alert_type) if @alert_type
84
+ send_event(msg, hash)
85
+ end
86
+
87
+ # Dogapi::Client will raise an error if unsuccessful.
88
+ def send_event(msg, hash)
89
+ client = Dogapi::Client.new(@api_key)
90
+ event = Dogapi::Event.new(msg, hash)
91
+ client.emit_event(event)
92
+ end
93
+
94
+ # set alert type
95
+ def default_alert_type(status)
96
+ case status
97
+ when :success then "success"
98
+ when :warning then "warning"
99
+ when :failure then "error"
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,99 @@
1
+ require "flowdock"
2
+
3
+ module Backup
4
+ module Notifier
5
+ class FlowDock < Base
6
+ ##
7
+ # The Flowdock API token
8
+ attr_accessor :token
9
+
10
+ ##
11
+ # Who the notification should appear from
12
+ attr_accessor :from_name
13
+
14
+ # Which email the notification should appear from
15
+ attr_accessor :from_email
16
+
17
+ ##
18
+ # source for message
19
+ attr_accessor :source
20
+
21
+ ##
22
+ # Subject for message
23
+ attr_accessor :subject
24
+
25
+ ##
26
+ # tag message in inbox
27
+ attr_accessor :tags
28
+
29
+ ##
30
+ # link for message
31
+ attr_accessor :link
32
+
33
+ def initialize(model, &block)
34
+ super
35
+ instance_eval(&block) if block_given?
36
+
37
+ @subject ||= default_subject
38
+ @source ||= default_source
39
+ @tags ||= []
40
+ end
41
+
42
+ private
43
+
44
+ ##
45
+ # Notify the user of the backup operation results.
46
+ #
47
+ # `status` indicates one of the following:
48
+ #
49
+ # `:success`
50
+ # : The backup completed successfully.
51
+ # : Notification will be sent if `on_success` is `true`.
52
+ #
53
+ # `:warning`
54
+ # : The backup completed successfully, but warnings were logged.
55
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
56
+ #
57
+ # `:failure`
58
+ # : The backup operation failed.
59
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
60
+ #
61
+ def notify!(status)
62
+ @tags += default_tags(status)
63
+ send_message(message.call(model, status: status_data_for(status)))
64
+ end
65
+
66
+ # Flowdock::Client will raise an error if unsuccessful.
67
+ def send_message(msg)
68
+ client = Flowdock::Flow.new(
69
+ api_token: token, source: source,
70
+ from: { name: from_name, address: from_email }
71
+ )
72
+
73
+ client.push_to_team_inbox(subject: subject,
74
+ content: msg,
75
+ tags: tags,
76
+ link: link)
77
+ end
78
+
79
+ # set related tags
80
+ def default_tags(status)
81
+ case status
82
+ when :success then ["#BackupSuccess"]
83
+ when :warning then ["#BackupWarning"]
84
+ when :failure then ["#BackupFailure"]
85
+ end
86
+ end
87
+
88
+ # set default source
89
+ def default_source
90
+ "Backup #{model.label}"
91
+ end
92
+
93
+ # set default subject
94
+ def default_subject
95
+ "Backup Notification"
96
+ end
97
+ end
98
+ end
99
+ end