backup-bouchard 4.4.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (134) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +19 -0
  3. data/README.md +29 -0
  4. data/bin/backup +5 -0
  5. data/lib/backup.rb +140 -0
  6. data/lib/backup/archive.rb +169 -0
  7. data/lib/backup/binder.rb +18 -0
  8. data/lib/backup/cleaner.rb +112 -0
  9. data/lib/backup/cli.rb +370 -0
  10. data/lib/backup/cloud_io/base.rb +38 -0
  11. data/lib/backup/cloud_io/cloud_files.rb +296 -0
  12. data/lib/backup/cloud_io/s3.rb +253 -0
  13. data/lib/backup/compressor/base.rb +32 -0
  14. data/lib/backup/compressor/bzip2.rb +35 -0
  15. data/lib/backup/compressor/custom.rb +49 -0
  16. data/lib/backup/compressor/gzip.rb +73 -0
  17. data/lib/backup/config.rb +118 -0
  18. data/lib/backup/config/dsl.rb +100 -0
  19. data/lib/backup/config/helpers.rb +137 -0
  20. data/lib/backup/database/base.rb +86 -0
  21. data/lib/backup/database/mongodb.rb +187 -0
  22. data/lib/backup/database/mysql.rb +191 -0
  23. data/lib/backup/database/openldap.rb +93 -0
  24. data/lib/backup/database/postgresql.rb +132 -0
  25. data/lib/backup/database/redis.rb +177 -0
  26. data/lib/backup/database/riak.rb +79 -0
  27. data/lib/backup/database/sqlite.rb +55 -0
  28. data/lib/backup/encryptor/base.rb +27 -0
  29. data/lib/backup/encryptor/gpg.rb +740 -0
  30. data/lib/backup/encryptor/open_ssl.rb +74 -0
  31. data/lib/backup/errors.rb +53 -0
  32. data/lib/backup/logger.rb +197 -0
  33. data/lib/backup/logger/console.rb +48 -0
  34. data/lib/backup/logger/fog_adapter.rb +25 -0
  35. data/lib/backup/logger/logfile.rb +131 -0
  36. data/lib/backup/logger/syslog.rb +114 -0
  37. data/lib/backup/model.rb +477 -0
  38. data/lib/backup/notifier/base.rb +126 -0
  39. data/lib/backup/notifier/campfire.rb +61 -0
  40. data/lib/backup/notifier/command.rb +99 -0
  41. data/lib/backup/notifier/datadog.rb +104 -0
  42. data/lib/backup/notifier/flowdock.rb +99 -0
  43. data/lib/backup/notifier/hipchat.rb +116 -0
  44. data/lib/backup/notifier/http_post.rb +114 -0
  45. data/lib/backup/notifier/mail.rb +246 -0
  46. data/lib/backup/notifier/nagios.rb +65 -0
  47. data/lib/backup/notifier/pagerduty.rb +79 -0
  48. data/lib/backup/notifier/prowl.rb +68 -0
  49. data/lib/backup/notifier/pushover.rb +71 -0
  50. data/lib/backup/notifier/ses.rb +103 -0
  51. data/lib/backup/notifier/slack.rb +147 -0
  52. data/lib/backup/notifier/twitter.rb +55 -0
  53. data/lib/backup/notifier/zabbix.rb +60 -0
  54. data/lib/backup/package.rb +51 -0
  55. data/lib/backup/packager.rb +105 -0
  56. data/lib/backup/pipeline.rb +120 -0
  57. data/lib/backup/splitter.rb +73 -0
  58. data/lib/backup/storage/base.rb +66 -0
  59. data/lib/backup/storage/cloud_files.rb +156 -0
  60. data/lib/backup/storage/cycler.rb +70 -0
  61. data/lib/backup/storage/dropbox.rb +210 -0
  62. data/lib/backup/storage/ftp.rb +110 -0
  63. data/lib/backup/storage/local.rb +61 -0
  64. data/lib/backup/storage/qiniu.rb +65 -0
  65. data/lib/backup/storage/rsync.rb +246 -0
  66. data/lib/backup/storage/s3.rb +155 -0
  67. data/lib/backup/storage/scp.rb +65 -0
  68. data/lib/backup/storage/sftp.rb +80 -0
  69. data/lib/backup/syncer/base.rb +67 -0
  70. data/lib/backup/syncer/cloud/base.rb +176 -0
  71. data/lib/backup/syncer/cloud/cloud_files.rb +81 -0
  72. data/lib/backup/syncer/cloud/local_file.rb +97 -0
  73. data/lib/backup/syncer/cloud/s3.rb +109 -0
  74. data/lib/backup/syncer/rsync/base.rb +50 -0
  75. data/lib/backup/syncer/rsync/local.rb +27 -0
  76. data/lib/backup/syncer/rsync/pull.rb +47 -0
  77. data/lib/backup/syncer/rsync/push.rb +201 -0
  78. data/lib/backup/template.rb +41 -0
  79. data/lib/backup/utilities.rb +228 -0
  80. data/lib/backup/version.rb +3 -0
  81. data/templates/cli/archive +28 -0
  82. data/templates/cli/compressor/bzip2 +4 -0
  83. data/templates/cli/compressor/custom +7 -0
  84. data/templates/cli/compressor/gzip +4 -0
  85. data/templates/cli/config +123 -0
  86. data/templates/cli/databases/mongodb +15 -0
  87. data/templates/cli/databases/mysql +18 -0
  88. data/templates/cli/databases/openldap +24 -0
  89. data/templates/cli/databases/postgresql +16 -0
  90. data/templates/cli/databases/redis +16 -0
  91. data/templates/cli/databases/riak +17 -0
  92. data/templates/cli/databases/sqlite +11 -0
  93. data/templates/cli/encryptor/gpg +27 -0
  94. data/templates/cli/encryptor/openssl +9 -0
  95. data/templates/cli/model +26 -0
  96. data/templates/cli/notifier/zabbix +15 -0
  97. data/templates/cli/notifiers/campfire +12 -0
  98. data/templates/cli/notifiers/command +32 -0
  99. data/templates/cli/notifiers/datadog +57 -0
  100. data/templates/cli/notifiers/flowdock +16 -0
  101. data/templates/cli/notifiers/hipchat +16 -0
  102. data/templates/cli/notifiers/http_post +32 -0
  103. data/templates/cli/notifiers/mail +24 -0
  104. data/templates/cli/notifiers/nagios +13 -0
  105. data/templates/cli/notifiers/pagerduty +12 -0
  106. data/templates/cli/notifiers/prowl +11 -0
  107. data/templates/cli/notifiers/pushover +11 -0
  108. data/templates/cli/notifiers/ses +15 -0
  109. data/templates/cli/notifiers/slack +22 -0
  110. data/templates/cli/notifiers/twitter +13 -0
  111. data/templates/cli/splitter +7 -0
  112. data/templates/cli/storages/cloud_files +11 -0
  113. data/templates/cli/storages/dropbox +20 -0
  114. data/templates/cli/storages/ftp +13 -0
  115. data/templates/cli/storages/local +8 -0
  116. data/templates/cli/storages/qiniu +12 -0
  117. data/templates/cli/storages/rsync +17 -0
  118. data/templates/cli/storages/s3 +16 -0
  119. data/templates/cli/storages/scp +15 -0
  120. data/templates/cli/storages/sftp +15 -0
  121. data/templates/cli/syncers/cloud_files +22 -0
  122. data/templates/cli/syncers/rsync_local +20 -0
  123. data/templates/cli/syncers/rsync_pull +28 -0
  124. data/templates/cli/syncers/rsync_push +28 -0
  125. data/templates/cli/syncers/s3 +27 -0
  126. data/templates/general/links +3 -0
  127. data/templates/general/version.erb +2 -0
  128. data/templates/notifier/mail/failure.erb +16 -0
  129. data/templates/notifier/mail/success.erb +16 -0
  130. data/templates/notifier/mail/warning.erb +16 -0
  131. data/templates/storage/dropbox/authorization_url.erb +6 -0
  132. data/templates/storage/dropbox/authorized.erb +4 -0
  133. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  134. metadata +518 -0
@@ -0,0 +1,126 @@
1
+ module Backup
2
+ module Notifier
3
+ class Error < Backup::Error; end
4
+
5
+ class Base
6
+ include Utilities::Helpers
7
+ include Config::Helpers
8
+
9
+ ##
10
+ # When set to true, the user will be notified by email
11
+ # when a backup process ends without raising any exceptions
12
+ attr_accessor :on_success
13
+ alias :notify_on_success? :on_success
14
+
15
+ ##
16
+ # When set to true, the user will be notified by email
17
+ # when a backup process is successful, but has warnings
18
+ attr_accessor :on_warning
19
+ alias :notify_on_warning? :on_warning
20
+
21
+ ##
22
+ # When set to true, the user will be notified by email
23
+ # when a backup process raises an exception before finishing
24
+ attr_accessor :on_failure
25
+ alias :notify_on_failure? :on_failure
26
+
27
+ ##
28
+ # Number of times to retry failed attempts to send notification.
29
+ # Default: 10
30
+ attr_accessor :max_retries
31
+
32
+ ##
33
+ # Time in seconds to pause before each retry.
34
+ # Default: 30
35
+ attr_accessor :retry_waitsec
36
+
37
+ ##
38
+ # Message to send. Depends on notifier implementation if this is used.
39
+ # Default: lambda returning:
40
+ # "#{ message } #{ model.label } (#{ model.trigger })"
41
+ #
42
+ # @yieldparam [model] Backup::Model
43
+ # @yieldparam [data] Hash containing `message` and `key` values.
44
+ attr_accessor :message
45
+
46
+ attr_reader :model
47
+
48
+ def initialize(model)
49
+ @model = model
50
+ load_defaults!
51
+
52
+ @on_success = true if on_success.nil?
53
+ @on_warning = true if on_warning.nil?
54
+ @on_failure = true if on_failure.nil?
55
+ @max_retries ||= 10
56
+ @retry_waitsec ||= 30
57
+ @message ||= lambda do |m, data|
58
+ "[#{data[:status][:message]}] #{m.label} (#{m.trigger})"
59
+ end
60
+ end
61
+
62
+ # This method is called from an ensure block in Model#perform! and must
63
+ # not raise any exceptions. However, each Notifier's #notify! method
64
+ # should raise an exception if the request fails so it may be retried.
65
+ def perform!
66
+ status =
67
+ case model.exit_status
68
+ when 0
69
+ :success if notify_on_success?
70
+ when 1
71
+ :warning if notify_on_success? || notify_on_warning?
72
+ else
73
+ :failure if notify_on_failure?
74
+ end
75
+
76
+ if status
77
+ Logger.info "Sending notification using #{notifier_name}..."
78
+ with_retries { notify!(status) }
79
+ end
80
+ rescue Exception => err
81
+ Logger.error Error.wrap(err, "#{notifier_name} Failed!")
82
+ end
83
+
84
+ private
85
+
86
+ def with_retries
87
+ retries = 0
88
+ begin
89
+ yield
90
+ rescue StandardError, Timeout::Error => err
91
+ retries += 1
92
+ raise if retries > max_retries
93
+
94
+ Logger.info Error.wrap(err, "Retry ##{retries} of #{max_retries}.")
95
+ sleep(retry_waitsec)
96
+ retry
97
+ end
98
+ end
99
+
100
+ ##
101
+ # Return the notifier name, with Backup namespace removed
102
+ def notifier_name
103
+ self.class.to_s.sub("Backup::", "")
104
+ end
105
+
106
+ ##
107
+ # Return status data for message creation
108
+ def status_data_for(status)
109
+ {
110
+ success: {
111
+ message: "Backup::Success",
112
+ key: :success
113
+ },
114
+ warning: {
115
+ message: "Backup::Warning",
116
+ key: :warning
117
+ },
118
+ failure: {
119
+ message: "Backup::Failure",
120
+ key: :failure
121
+ }
122
+ }[status]
123
+ end
124
+ end
125
+ end
126
+ end
@@ -0,0 +1,61 @@
1
+ require "json"
2
+
3
+ module Backup
4
+ module Notifier
5
+ class Campfire < Base
6
+ ##
7
+ # Campfire api authentication token
8
+ attr_accessor :api_token
9
+
10
+ ##
11
+ # Campfire account's subdomain
12
+ attr_accessor :subdomain
13
+
14
+ ##
15
+ # Campfire account's room id
16
+ attr_accessor :room_id
17
+
18
+ def initialize(model, &block)
19
+ super
20
+ instance_eval(&block) if block_given?
21
+ end
22
+
23
+ private
24
+
25
+ ##
26
+ # Notify the user of the backup operation results.
27
+ #
28
+ # `status` indicates one of the following:
29
+ #
30
+ # `:success`
31
+ # : The backup completed successfully.
32
+ # : Notification will be sent if `on_success` is `true`.
33
+ #
34
+ # `:warning`
35
+ # : The backup completed successfully, but warnings were logged.
36
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
37
+ #
38
+ # `:failure`
39
+ # : The backup operation failed.
40
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
41
+ #
42
+ def notify!(status)
43
+ send_message(message.call(model, status: status_data_for(status)))
44
+ end
45
+
46
+ def send_message(message)
47
+ uri = "https://#{subdomain}.campfirenow.com/room/#{room_id}/speak.json"
48
+ options = {
49
+ headers: { "Content-Type" => "application/json" },
50
+ body: JSON.dump(
51
+ message: { body: message, type: "Textmessage" }
52
+ )
53
+ }
54
+ options[:user] = api_token
55
+ options[:password] = "x" # Basic Auth
56
+ options[:expects] = 201 # raise error if unsuccessful
57
+ Excon.post(uri, options)
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,99 @@
1
+ module Backup
2
+ module Notifier
3
+ class Command < Base
4
+ ##
5
+ # Command to execute.
6
+ #
7
+ # Make sure it is accessible from your $PATH, or provide
8
+ # the absolute path to the command.
9
+ attr_accessor :command
10
+
11
+ ##
12
+ # Arguments to pass to the command.
13
+ #
14
+ # Must be an array of strings or callable objects.
15
+ #
16
+ # Callables will be invoked with #call(model, status),
17
+ # and the return value used as the argument.
18
+ #
19
+ # In strings you can use the following placeholders:
20
+ #
21
+ # %l - Model label
22
+ # %t - Model trigger
23
+ # %d - Backup duration (HH:MM:SS)
24
+ # %s - Status (success/failure/warning)
25
+ # %v - Status verb (succeeded/failed/succeeded with warnings)
26
+ #
27
+ # All placeholders can be used with uppercase letters to capitalize
28
+ # the value.
29
+ #
30
+ # Defaults to ["%L %v"]
31
+ attr_accessor :args
32
+
33
+ def initialize(model, &block)
34
+ super
35
+ instance_eval(&block) if block_given?
36
+
37
+ @args ||= ["%L %v"]
38
+ end
39
+
40
+ private
41
+
42
+ ##
43
+ # Notify the user of the backup operation results.
44
+ #
45
+ # `status` indicates one of the following:
46
+ #
47
+ # `:success`
48
+ # : The backup completed successfully.
49
+ # : Notification will be sent if `on_success` is `true`.
50
+ #
51
+ # `:warning`
52
+ # : The backup completed successfully, but warnings were logged.
53
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
54
+ #
55
+ # `:failure`
56
+ # : The backup operation failed.
57
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
58
+ #
59
+ def notify!(status)
60
+ IO.popen([@command] + args.map { |arg| format_arg(arg, status) })
61
+ end
62
+
63
+ def format_arg(arg, status)
64
+ if arg.respond_to?(:call)
65
+ arg.call(model, status)
66
+ else
67
+ arg.gsub(/%(\w)/) do |match|
68
+ ph = match[1]
69
+ val = case ph.downcase
70
+ when "l"
71
+ model.label
72
+ when "t"
73
+ model.trigger.to_s
74
+ when "d"
75
+ model.duration
76
+ when "v"
77
+ status_verb(status)
78
+ when "s"
79
+ status.to_s
80
+ end
81
+ val.capitalize! if ph == ph.upcase
82
+ val
83
+ end
84
+ end
85
+ end
86
+
87
+ def status_verb(status)
88
+ case status
89
+ when :success
90
+ "succeeded"
91
+ when :failure
92
+ "failed"
93
+ when :warning
94
+ "succeeded with warnings"
95
+ end
96
+ end
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,104 @@
1
+ require "dogapi"
2
+
3
+ module Backup
4
+ module Notifier
5
+ class DataDog < Base
6
+ ##
7
+ # The DataDog API key
8
+ attr_accessor :api_key
9
+
10
+ ##
11
+ # The title of the event
12
+ attr_accessor :title
13
+
14
+ attr_deprecate :text,
15
+ version: "4.2",
16
+ message: "Please use the `message` attribute. For more information "\
17
+ "see https://github.com/backup/backup/pull/698"
18
+
19
+ ##
20
+ # The timestamp for the event
21
+ attr_accessor :date_happened
22
+
23
+ ##
24
+ # The priority of the event (low/normal)
25
+ attr_accessor :priority
26
+
27
+ ##
28
+ # The host that generated the event
29
+ attr_accessor :host
30
+
31
+ ##
32
+ # The tags for this host (should be an array)
33
+ attr_accessor :tags
34
+
35
+ ##
36
+ # The alert_type of the event (error/warning/info/success)
37
+ attr_accessor :alert_type
38
+
39
+ ##
40
+ # The aggregation_key for the event
41
+ attr_accessor :aggregation_key
42
+
43
+ ##
44
+ # The source_type for the event (nagios, hudson, jenkins, user, my apps, feed, chef, puppet, git, bitbucket, fabric, capistrano)
45
+ attr_accessor :source_type_name
46
+
47
+ def initialize(model, &block)
48
+ super
49
+ instance_eval(&block) if block_given?
50
+ @title ||= "Backup #{model.label}"
51
+ end
52
+
53
+ private
54
+
55
+ ##
56
+ # Notify the user of the backup operation results.
57
+ #
58
+ # `status` indicates one of the following:
59
+ #
60
+ # `:success`
61
+ # : The backup completed successfully.
62
+ # : Notification will be sent if `on_success` is `true`.
63
+ #
64
+ # `:warning`
65
+ # : The backup completed successfully, but warnings were logged.
66
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
67
+ #
68
+ # `:failure`
69
+ # : The backup operation failed.
70
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
71
+ #
72
+ def notify!(status)
73
+ msg = message.call(model, status: status_data_for(status))
74
+
75
+ hash = { alert_type: default_alert_type(status) }
76
+ hash.store(:msg_title, @title)
77
+ hash.store(:date_happened, @date_happened) if @date_happened
78
+ hash.store(:priority, @priority) if @priority
79
+ hash.store(:host, @host) if @host
80
+ hash.store(:tags, @tags) if @tags
81
+ hash.store(:aggregation_key, @aggregation_key) if @aggregation_key
82
+ hash.store(:source_type_name, @source_type_name) if @source_type_name
83
+ hash.store(:alert_type, @alert_type) if @alert_type
84
+ send_event(msg, hash)
85
+ end
86
+
87
+ # Dogapi::Client will raise an error if unsuccessful.
88
+ def send_event(msg, hash)
89
+ client = Dogapi::Client.new(@api_key)
90
+ event = Dogapi::Event.new(msg, hash)
91
+ client.emit_event(event)
92
+ end
93
+
94
+ # set alert type
95
+ def default_alert_type(status)
96
+ case status
97
+ when :success then "success"
98
+ when :warning then "warning"
99
+ when :failure then "error"
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,99 @@
1
+ require "flowdock"
2
+
3
+ module Backup
4
+ module Notifier
5
+ class FlowDock < Base
6
+ ##
7
+ # The Flowdock API token
8
+ attr_accessor :token
9
+
10
+ ##
11
+ # Who the notification should appear from
12
+ attr_accessor :from_name
13
+
14
+ # Which email the notification should appear from
15
+ attr_accessor :from_email
16
+
17
+ ##
18
+ # source for message
19
+ attr_accessor :source
20
+
21
+ ##
22
+ # Subject for message
23
+ attr_accessor :subject
24
+
25
+ ##
26
+ # tag message in inbox
27
+ attr_accessor :tags
28
+
29
+ ##
30
+ # link for message
31
+ attr_accessor :link
32
+
33
+ def initialize(model, &block)
34
+ super
35
+ instance_eval(&block) if block_given?
36
+
37
+ @subject ||= default_subject
38
+ @source ||= default_source
39
+ @tags ||= []
40
+ end
41
+
42
+ private
43
+
44
+ ##
45
+ # Notify the user of the backup operation results.
46
+ #
47
+ # `status` indicates one of the following:
48
+ #
49
+ # `:success`
50
+ # : The backup completed successfully.
51
+ # : Notification will be sent if `on_success` is `true`.
52
+ #
53
+ # `:warning`
54
+ # : The backup completed successfully, but warnings were logged.
55
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
56
+ #
57
+ # `:failure`
58
+ # : The backup operation failed.
59
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
60
+ #
61
+ def notify!(status)
62
+ @tags += default_tags(status)
63
+ send_message(message.call(model, status: status_data_for(status)))
64
+ end
65
+
66
+ # Flowdock::Client will raise an error if unsuccessful.
67
+ def send_message(msg)
68
+ client = Flowdock::Flow.new(
69
+ api_token: token, source: source,
70
+ from: { name: from_name, address: from_email }
71
+ )
72
+
73
+ client.push_to_team_inbox(subject: subject,
74
+ content: msg,
75
+ tags: tags,
76
+ link: link)
77
+ end
78
+
79
+ # set related tags
80
+ def default_tags(status)
81
+ case status
82
+ when :success then ["#BackupSuccess"]
83
+ when :warning then ["#BackupWarning"]
84
+ when :failure then ["#BackupFailure"]
85
+ end
86
+ end
87
+
88
+ # set default source
89
+ def default_source
90
+ "Backup #{model.label}"
91
+ end
92
+
93
+ # set default subject
94
+ def default_subject
95
+ "Backup Notification"
96
+ end
97
+ end
98
+ end
99
+ end