cm-backup 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +20 -0
  3. data/bin/backup +5 -0
  4. data/lib/backup.rb +144 -0
  5. data/lib/backup/archive.rb +170 -0
  6. data/lib/backup/binder.rb +22 -0
  7. data/lib/backup/cleaner.rb +116 -0
  8. data/lib/backup/cli.rb +374 -0
  9. data/lib/backup/cloud_io/base.rb +41 -0
  10. data/lib/backup/cloud_io/cloud_files.rb +298 -0
  11. data/lib/backup/cloud_io/s3.rb +260 -0
  12. data/lib/backup/compressor/base.rb +35 -0
  13. data/lib/backup/compressor/bzip2.rb +39 -0
  14. data/lib/backup/compressor/custom.rb +53 -0
  15. data/lib/backup/compressor/gzip.rb +74 -0
  16. data/lib/backup/config.rb +119 -0
  17. data/lib/backup/config/dsl.rb +103 -0
  18. data/lib/backup/config/helpers.rb +143 -0
  19. data/lib/backup/database/base.rb +85 -0
  20. data/lib/backup/database/mongodb.rb +187 -0
  21. data/lib/backup/database/mysql.rb +192 -0
  22. data/lib/backup/database/openldap.rb +95 -0
  23. data/lib/backup/database/postgresql.rb +133 -0
  24. data/lib/backup/database/redis.rb +179 -0
  25. data/lib/backup/database/riak.rb +82 -0
  26. data/lib/backup/database/sqlite.rb +57 -0
  27. data/lib/backup/encryptor/base.rb +29 -0
  28. data/lib/backup/encryptor/gpg.rb +747 -0
  29. data/lib/backup/encryptor/open_ssl.rb +77 -0
  30. data/lib/backup/errors.rb +58 -0
  31. data/lib/backup/logger.rb +199 -0
  32. data/lib/backup/logger/console.rb +51 -0
  33. data/lib/backup/logger/fog_adapter.rb +29 -0
  34. data/lib/backup/logger/logfile.rb +133 -0
  35. data/lib/backup/logger/syslog.rb +116 -0
  36. data/lib/backup/model.rb +479 -0
  37. data/lib/backup/notifier/base.rb +128 -0
  38. data/lib/backup/notifier/campfire.rb +63 -0
  39. data/lib/backup/notifier/command.rb +102 -0
  40. data/lib/backup/notifier/datadog.rb +107 -0
  41. data/lib/backup/notifier/flowdock.rb +103 -0
  42. data/lib/backup/notifier/hipchat.rb +118 -0
  43. data/lib/backup/notifier/http_post.rb +117 -0
  44. data/lib/backup/notifier/mail.rb +249 -0
  45. data/lib/backup/notifier/nagios.rb +69 -0
  46. data/lib/backup/notifier/pagerduty.rb +81 -0
  47. data/lib/backup/notifier/prowl.rb +68 -0
  48. data/lib/backup/notifier/pushover.rb +74 -0
  49. data/lib/backup/notifier/ses.rb +105 -0
  50. data/lib/backup/notifier/slack.rb +148 -0
  51. data/lib/backup/notifier/twitter.rb +58 -0
  52. data/lib/backup/notifier/zabbix.rb +63 -0
  53. data/lib/backup/package.rb +55 -0
  54. data/lib/backup/packager.rb +107 -0
  55. data/lib/backup/pipeline.rb +124 -0
  56. data/lib/backup/splitter.rb +76 -0
  57. data/lib/backup/storage/base.rb +69 -0
  58. data/lib/backup/storage/cloud_files.rb +158 -0
  59. data/lib/backup/storage/cycler.rb +75 -0
  60. data/lib/backup/storage/dropbox.rb +212 -0
  61. data/lib/backup/storage/ftp.rb +112 -0
  62. data/lib/backup/storage/local.rb +64 -0
  63. data/lib/backup/storage/qiniu.rb +65 -0
  64. data/lib/backup/storage/rsync.rb +248 -0
  65. data/lib/backup/storage/s3.rb +156 -0
  66. data/lib/backup/storage/scp.rb +67 -0
  67. data/lib/backup/storage/sftp.rb +82 -0
  68. data/lib/backup/syncer/base.rb +70 -0
  69. data/lib/backup/syncer/cloud/base.rb +179 -0
  70. data/lib/backup/syncer/cloud/cloud_files.rb +83 -0
  71. data/lib/backup/syncer/cloud/local_file.rb +100 -0
  72. data/lib/backup/syncer/cloud/s3.rb +110 -0
  73. data/lib/backup/syncer/rsync/base.rb +54 -0
  74. data/lib/backup/syncer/rsync/local.rb +31 -0
  75. data/lib/backup/syncer/rsync/pull.rb +51 -0
  76. data/lib/backup/syncer/rsync/push.rb +205 -0
  77. data/lib/backup/template.rb +46 -0
  78. data/lib/backup/utilities.rb +224 -0
  79. data/lib/backup/version.rb +5 -0
  80. data/templates/cli/archive +28 -0
  81. data/templates/cli/compressor/bzip2 +4 -0
  82. data/templates/cli/compressor/custom +7 -0
  83. data/templates/cli/compressor/gzip +4 -0
  84. data/templates/cli/config +123 -0
  85. data/templates/cli/databases/mongodb +15 -0
  86. data/templates/cli/databases/mysql +18 -0
  87. data/templates/cli/databases/openldap +24 -0
  88. data/templates/cli/databases/postgresql +16 -0
  89. data/templates/cli/databases/redis +16 -0
  90. data/templates/cli/databases/riak +17 -0
  91. data/templates/cli/databases/sqlite +11 -0
  92. data/templates/cli/encryptor/gpg +27 -0
  93. data/templates/cli/encryptor/openssl +9 -0
  94. data/templates/cli/model +26 -0
  95. data/templates/cli/notifier/zabbix +15 -0
  96. data/templates/cli/notifiers/campfire +12 -0
  97. data/templates/cli/notifiers/command +32 -0
  98. data/templates/cli/notifiers/datadog +57 -0
  99. data/templates/cli/notifiers/flowdock +16 -0
  100. data/templates/cli/notifiers/hipchat +16 -0
  101. data/templates/cli/notifiers/http_post +32 -0
  102. data/templates/cli/notifiers/mail +24 -0
  103. data/templates/cli/notifiers/nagios +13 -0
  104. data/templates/cli/notifiers/pagerduty +12 -0
  105. data/templates/cli/notifiers/prowl +11 -0
  106. data/templates/cli/notifiers/pushover +11 -0
  107. data/templates/cli/notifiers/ses +15 -0
  108. data/templates/cli/notifiers/slack +22 -0
  109. data/templates/cli/notifiers/twitter +13 -0
  110. data/templates/cli/splitter +7 -0
  111. data/templates/cli/storages/cloud_files +11 -0
  112. data/templates/cli/storages/dropbox +20 -0
  113. data/templates/cli/storages/ftp +13 -0
  114. data/templates/cli/storages/local +8 -0
  115. data/templates/cli/storages/qiniu +12 -0
  116. data/templates/cli/storages/rsync +17 -0
  117. data/templates/cli/storages/s3 +16 -0
  118. data/templates/cli/storages/scp +15 -0
  119. data/templates/cli/storages/sftp +15 -0
  120. data/templates/cli/syncers/cloud_files +22 -0
  121. data/templates/cli/syncers/rsync_local +20 -0
  122. data/templates/cli/syncers/rsync_pull +28 -0
  123. data/templates/cli/syncers/rsync_push +28 -0
  124. data/templates/cli/syncers/s3 +27 -0
  125. data/templates/general/links +3 -0
  126. data/templates/general/version.erb +2 -0
  127. data/templates/notifier/mail/failure.erb +16 -0
  128. data/templates/notifier/mail/success.erb +16 -0
  129. data/templates/notifier/mail/warning.erb +16 -0
  130. data/templates/storage/dropbox/authorization_url.erb +6 -0
  131. data/templates/storage/dropbox/authorized.erb +4 -0
  132. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  133. metadata +1077 -0
@@ -0,0 +1,128 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Notifier
5
+ class Error < Backup::Error; end
6
+
7
+ class Base
8
+ include Utilities::Helpers
9
+ include Config::Helpers
10
+
11
+ ##
12
+ # When set to true, the user will be notified by email
13
+ # when a backup process ends without raising any exceptions
14
+ attr_accessor :on_success
15
+ alias :notify_on_success? :on_success
16
+
17
+ ##
18
+ # When set to true, the user will be notified by email
19
+ # when a backup process is successful, but has warnings
20
+ attr_accessor :on_warning
21
+ alias :notify_on_warning? :on_warning
22
+
23
+ ##
24
+ # When set to true, the user will be notified by email
25
+ # when a backup process raises an exception before finishing
26
+ attr_accessor :on_failure
27
+ alias :notify_on_failure? :on_failure
28
+
29
+ ##
30
+ # Number of times to retry failed attempts to send notification.
31
+ # Default: 10
32
+ attr_accessor :max_retries
33
+
34
+ ##
35
+ # Time in seconds to pause before each retry.
36
+ # Default: 30
37
+ attr_accessor :retry_waitsec
38
+
39
+ ##
40
+ # Message to send. Depends on notifier implementation if this is used.
41
+ # Default: lambda returning:
42
+ # "#{ message } #{ model.label } (#{ model.trigger })"
43
+ #
44
+ # @yieldparam [model] Backup::Model
45
+ # @yieldparam [data] Hash containing `message` and `key` values.
46
+ attr_accessor :message
47
+
48
+ attr_reader :model
49
+
50
+ def initialize(model)
51
+ @model = model
52
+ load_defaults!
53
+
54
+ @on_success = true if on_success.nil?
55
+ @on_warning = true if on_warning.nil?
56
+ @on_failure = true if on_failure.nil?
57
+ @max_retries ||= 10
58
+ @retry_waitsec ||= 30
59
+ @message ||= lambda do |model, data|
60
+ "[#{ data[:status][:message] }] #{ model.label } (#{ model.trigger })"
61
+ end
62
+ end
63
+
64
+ # This method is called from an ensure block in Model#perform! and must
65
+ # not raise any exceptions. However, each Notifier's #notify! method
66
+ # should raise an exception if the request fails so it may be retried.
67
+ def perform!
68
+ status = case model.exit_status
69
+ when 0
70
+ :success if notify_on_success?
71
+ when 1
72
+ :warning if notify_on_success? || notify_on_warning?
73
+ else
74
+ :failure if notify_on_failure?
75
+ end
76
+
77
+ if status
78
+ Logger.info "Sending notification using #{ notifier_name }..."
79
+ with_retries { notify!(status) }
80
+ end
81
+
82
+ rescue Exception => err
83
+ Logger.error Error.wrap(err, "#{ notifier_name } Failed!")
84
+ end
85
+
86
+ private
87
+
88
+ def with_retries
89
+ retries = 0
90
+ begin
91
+ yield
92
+ rescue StandardError, Timeout::Error => err
93
+ retries += 1
94
+ raise if retries > max_retries
95
+
96
+ Logger.info Error.wrap(err, "Retry ##{ retries } of #{ max_retries }.")
97
+ sleep(retry_waitsec)
98
+ retry
99
+ end
100
+ end
101
+
102
+ ##
103
+ # Return the notifier name, with Backup namespace removed
104
+ def notifier_name
105
+ self.class.to_s.sub('Backup::', '')
106
+ end
107
+
108
+ ##
109
+ # Return status data for message creation
110
+ def status_data_for(status)
111
+ {
112
+ :success => {
113
+ :message => 'Backup::Success',
114
+ :key => :success
115
+ },
116
+ :warning => {
117
+ :message => 'Backup::Warning',
118
+ :key => :warning
119
+ },
120
+ :failure => {
121
+ :message => 'Backup::Failure',
122
+ :key => :failure
123
+ }
124
+ }[status]
125
+ end
126
+ end
127
+ end
128
+ end
@@ -0,0 +1,63 @@
1
+ # encoding: utf-8
2
+ require 'json'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class Campfire < Base
7
+
8
+ ##
9
+ # Campfire api authentication token
10
+ attr_accessor :api_token
11
+
12
+ ##
13
+ # Campfire account's subdomain
14
+ attr_accessor :subdomain
15
+
16
+ ##
17
+ # Campfire account's room id
18
+ attr_accessor :room_id
19
+
20
+ def initialize(model, &block)
21
+ super
22
+ instance_eval(&block) if block_given?
23
+ end
24
+
25
+ private
26
+
27
+ ##
28
+ # Notify the user of the backup operation results.
29
+ #
30
+ # `status` indicates one of the following:
31
+ #
32
+ # `:success`
33
+ # : The backup completed successfully.
34
+ # : Notification will be sent if `on_success` is `true`.
35
+ #
36
+ # `:warning`
37
+ # : The backup completed successfully, but warnings were logged.
38
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
39
+ #
40
+ # `:failure`
41
+ # : The backup operation failed.
42
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
43
+ #
44
+ def notify!(status)
45
+ send_message(message.call(model, :status => status_data_for(status)))
46
+ end
47
+
48
+ def send_message(message)
49
+ uri = "https://#{ subdomain }.campfirenow.com/room/#{ room_id }/speak.json"
50
+ options = {
51
+ :headers => { 'Content-Type' => 'application/json' },
52
+ :body => JSON.dump(
53
+ { :message => { :body => message, :type => 'Textmessage' } }
54
+ )
55
+ }
56
+ options.merge!(:user => api_token, :password => 'x') # Basic Auth
57
+ options.merge!(:expects => 201) # raise error if unsuccessful
58
+ Excon.post(uri, options)
59
+ end
60
+
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,102 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Notifier
5
+ class Command < Base
6
+
7
+ ##
8
+ # Command to execute.
9
+ #
10
+ # Make sure it is accessible from your $PATH, or provide
11
+ # the absolute path to the command.
12
+ attr_accessor :command
13
+
14
+ ##
15
+ # Arguments to pass to the command.
16
+ #
17
+ # Must be an array of strings or callable objects.
18
+ #
19
+ # Callables will be invoked with #call(model, status),
20
+ # and the return value used as the argument.
21
+ #
22
+ # In strings you can use the following placeholders:
23
+ #
24
+ # %l - Model label
25
+ # %t - Model trigger
26
+ # %d - Backup duration (HH:MM:SS)
27
+ # %s - Status (success/failure/warning)
28
+ # %v - Status verb (succeeded/failed/succeeded with warnings)
29
+ #
30
+ # All placeholders can be used with uppercase letters to capitalize
31
+ # the value.
32
+ #
33
+ # Defaults to ["%L %v"]
34
+ attr_accessor :args
35
+
36
+ def initialize(model, &block)
37
+ super
38
+ instance_eval(&block) if block_given?
39
+
40
+ @args ||= ["%L %v"]
41
+ end
42
+
43
+ private
44
+
45
+ ##
46
+ # Notify the user of the backup operation results.
47
+ #
48
+ # `status` indicates one of the following:
49
+ #
50
+ # `:success`
51
+ # : The backup completed successfully.
52
+ # : Notification will be sent if `on_success` is `true`.
53
+ #
54
+ # `:warning`
55
+ # : The backup completed successfully, but warnings were logged.
56
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
57
+ #
58
+ # `:failure`
59
+ # : The backup operation failed.
60
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
61
+ #
62
+ def notify!(status)
63
+ IO.popen([@command] + args.map { |arg| format_arg(arg, status) })
64
+ end
65
+
66
+ def format_arg(arg, status)
67
+ if arg.respond_to?(:call)
68
+ arg.call(model, status)
69
+ else
70
+ arg.gsub(/%(\w)/) do |match|
71
+ ph = match[1]
72
+ val = case ph.downcase
73
+ when "l"
74
+ model.label
75
+ when "t"
76
+ model.trigger.to_s
77
+ when "d"
78
+ model.duration
79
+ when "v"
80
+ status_verb(status)
81
+ when "s"
82
+ status.to_s
83
+ end
84
+ val.capitalize! if ph == ph.upcase
85
+ val
86
+ end
87
+ end
88
+ end
89
+
90
+ def status_verb(status)
91
+ case status
92
+ when :success
93
+ "succeeded"
94
+ when :failure
95
+ "failed"
96
+ when :warning
97
+ "succeeded with warnings"
98
+ end
99
+ end
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,107 @@
1
+ # encoding: utf-8
2
+ require 'dogapi'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class DataDog < Base
7
+
8
+ ##
9
+ # The DataDog API key
10
+ attr_accessor :api_key
11
+
12
+ ##
13
+ # The title of the event
14
+ attr_accessor :title
15
+
16
+ attr_deprecate :text,
17
+ :version => '4.2',
18
+ :message => 'Please use the `message` attribute. For more information '\
19
+ 'see https://github.com/backup/backup/pull/698'
20
+
21
+ ##
22
+ # The timestamp for the event
23
+ attr_accessor :date_happened
24
+
25
+ ##
26
+ # The priority of the event (low/normal)
27
+ attr_accessor :priority
28
+
29
+ ##
30
+ # The host that generated the event
31
+ attr_accessor :host
32
+
33
+ ##
34
+ # The tags for this host (should be an array)
35
+ attr_accessor :tags
36
+
37
+ ##
38
+ # The alert_type of the event (error/warning/info/success)
39
+ attr_accessor :alert_type
40
+
41
+ ##
42
+ # The aggregation_key for the event
43
+ attr_accessor :aggregation_key
44
+
45
+ ##
46
+ # The source_type for the event (nagios, hudson, jenkins, user, my apps, feed, chef, puppet, git, bitbucket, fabric, capistrano)
47
+ attr_accessor :source_type_name
48
+
49
+ def initialize(model, &block)
50
+ super
51
+ instance_eval(&block) if block_given?
52
+ @title ||= "Backup #{ model.label }"
53
+ end
54
+
55
+ private
56
+
57
+ ##
58
+ # Notify the user of the backup operation results.
59
+ #
60
+ # `status` indicates one of the following:
61
+ #
62
+ # `:success`
63
+ # : The backup completed successfully.
64
+ # : Notification will be sent if `on_success` is `true`.
65
+ #
66
+ # `:warning`
67
+ # : The backup completed successfully, but warnings were logged.
68
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
69
+ #
70
+ # `:failure`
71
+ # : The backup operation failed.
72
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
73
+ #
74
+ def notify!(status)
75
+ msg = message.call(model, :status => status_data_for(status))
76
+
77
+ hash = { alert_type: default_alert_type(status) }
78
+ hash.store(:msg_title, @title)
79
+ hash.store(:date_happened, @date_happened) if @date_happened
80
+ hash.store(:priority, @priority) if @priority
81
+ hash.store(:host, @host) if @host
82
+ hash.store(:tags, @tags) if @tags
83
+ hash.store(:aggregation_key, @aggregation_key) if @aggregation_key
84
+ hash.store(:source_type_name, @source_type_name) if @source_type_name
85
+ hash.store(:alert_type, @alert_type) if @alert_type
86
+ send_event(msg, hash)
87
+ end
88
+
89
+ # Dogapi::Client will raise an error if unsuccessful.
90
+ def send_event(msg, hash)
91
+ client = Dogapi::Client.new(@api_key)
92
+ event = Dogapi::Event.new(msg, hash)
93
+ client.emit_event(event)
94
+ end
95
+
96
+ # set alert type
97
+ def default_alert_type(status)
98
+ case status
99
+ when :success then 'success'
100
+ when :warning then 'warning'
101
+ when :failure then 'error'
102
+ end
103
+ end
104
+
105
+ end
106
+ end
107
+ end
@@ -0,0 +1,103 @@
1
+ # encoding: utf-8
2
+ require 'flowdock'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class FlowDock < Base
7
+
8
+ ##
9
+ # The Flowdock API token
10
+ attr_accessor :token
11
+
12
+ ##
13
+ # Who the notification should appear from
14
+ attr_accessor :from_name
15
+
16
+ # Which email the notification should appear from
17
+ attr_accessor :from_email
18
+
19
+ ##
20
+ # source for message
21
+ attr_accessor :source
22
+
23
+ ##
24
+ # Subject for message
25
+ attr_accessor :subject
26
+
27
+ ##
28
+ # tag message in inbox
29
+ attr_accessor :tags
30
+
31
+ ##
32
+ # link for message
33
+ attr_accessor :link
34
+
35
+ def initialize(model, &block)
36
+ super
37
+ instance_eval(&block) if block_given?
38
+
39
+ @subject ||= default_subject
40
+ @source ||= default_source
41
+ @tags ||= []
42
+ end
43
+
44
+ private
45
+
46
+ ##
47
+ # Notify the user of the backup operation results.
48
+ #
49
+ # `status` indicates one of the following:
50
+ #
51
+ # `:success`
52
+ # : The backup completed successfully.
53
+ # : Notification will be sent if `on_success` is `true`.
54
+ #
55
+ # `:warning`
56
+ # : The backup completed successfully, but warnings were logged.
57
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
58
+ #
59
+ # `:failure`
60
+ # : The backup operation failed.
61
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
62
+ #
63
+ def notify!(status)
64
+ @tags += default_tags(status)
65
+ send_message(message.call(model, status: status_data_for(status)))
66
+ end
67
+
68
+ # Flowdock::Client will raise an error if unsuccessful.
69
+ def send_message(msg)
70
+ client = Flowdock::Flow.new(
71
+ :api_token => token, :source => source,
72
+ :from => {:name => from_name, :address => from_email }
73
+ )
74
+
75
+ client.push_to_team_inbox(:subject => subject,
76
+ :content => msg,
77
+ :tags => tags,
78
+ :link => link )
79
+ end
80
+
81
+ # set related tags
82
+ def default_tags(status)
83
+ case status
84
+ when :success then ['#BackupSuccess']
85
+ when :warning then ['#BackupWarning']
86
+ when :failure then ['#BackupFailure']
87
+ end
88
+ end
89
+
90
+
91
+ #set default source
92
+ def default_source
93
+ "Backup #{ model.label }"
94
+ end
95
+
96
+ # set default subject
97
+ def default_subject
98
+ 'Backup Notification'
99
+ end
100
+
101
+ end
102
+ end
103
+ end