venet-backup 4.1.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (136) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.md +24 -0
  3. data/README.md +15 -0
  4. data/bin/backup +5 -0
  5. data/lib/backup.rb +141 -0
  6. data/lib/backup/archive.rb +170 -0
  7. data/lib/backup/binder.rb +22 -0
  8. data/lib/backup/cleaner.rb +116 -0
  9. data/lib/backup/cli.rb +374 -0
  10. data/lib/backup/cloud_io/base.rb +41 -0
  11. data/lib/backup/cloud_io/cloud_files.rb +298 -0
  12. data/lib/backup/cloud_io/s3.rb +260 -0
  13. data/lib/backup/compressor/base.rb +35 -0
  14. data/lib/backup/compressor/bzip2.rb +39 -0
  15. data/lib/backup/compressor/custom.rb +53 -0
  16. data/lib/backup/compressor/gzip.rb +74 -0
  17. data/lib/backup/config.rb +119 -0
  18. data/lib/backup/config/dsl.rb +103 -0
  19. data/lib/backup/config/helpers.rb +143 -0
  20. data/lib/backup/database/base.rb +85 -0
  21. data/lib/backup/database/mongodb.rb +186 -0
  22. data/lib/backup/database/mysql.rb +180 -0
  23. data/lib/backup/database/openldap.rb +95 -0
  24. data/lib/backup/database/postgresql.rb +133 -0
  25. data/lib/backup/database/redis.rb +179 -0
  26. data/lib/backup/database/riak.rb +82 -0
  27. data/lib/backup/database/sqlite.rb +57 -0
  28. data/lib/backup/encryptor/base.rb +29 -0
  29. data/lib/backup/encryptor/gpg.rb +747 -0
  30. data/lib/backup/encryptor/open_ssl.rb +72 -0
  31. data/lib/backup/errors.rb +58 -0
  32. data/lib/backup/logger.rb +199 -0
  33. data/lib/backup/logger/console.rb +51 -0
  34. data/lib/backup/logger/fog_adapter.rb +29 -0
  35. data/lib/backup/logger/logfile.rb +133 -0
  36. data/lib/backup/logger/syslog.rb +116 -0
  37. data/lib/backup/model.rb +454 -0
  38. data/lib/backup/notifier/base.rb +98 -0
  39. data/lib/backup/notifier/campfire.rb +69 -0
  40. data/lib/backup/notifier/datadog.rb +116 -0
  41. data/lib/backup/notifier/flowdock.rb +102 -0
  42. data/lib/backup/notifier/hipchat.rb +93 -0
  43. data/lib/backup/notifier/http_post.rb +122 -0
  44. data/lib/backup/notifier/mail.rb +238 -0
  45. data/lib/backup/notifier/nagios.rb +74 -0
  46. data/lib/backup/notifier/pagerduty.rb +81 -0
  47. data/lib/backup/notifier/prowl.rb +69 -0
  48. data/lib/backup/notifier/pushover.rb +80 -0
  49. data/lib/backup/notifier/slack.rb +158 -0
  50. data/lib/backup/notifier/twitter.rb +64 -0
  51. data/lib/backup/notifier/zabbix.rb +68 -0
  52. data/lib/backup/package.rb +51 -0
  53. data/lib/backup/packager.rb +101 -0
  54. data/lib/backup/pipeline.rb +124 -0
  55. data/lib/backup/splitter.rb +76 -0
  56. data/lib/backup/storage/base.rb +57 -0
  57. data/lib/backup/storage/cloud_files.rb +158 -0
  58. data/lib/backup/storage/cycler.rb +65 -0
  59. data/lib/backup/storage/dropbox.rb +236 -0
  60. data/lib/backup/storage/ftp.rb +98 -0
  61. data/lib/backup/storage/google/google_drive_auth.rb +96 -0
  62. data/lib/backup/storage/google/google_drive_transfer.rb +125 -0
  63. data/lib/backup/storage/google_drive.rb +62 -0
  64. data/lib/backup/storage/local.rb +64 -0
  65. data/lib/backup/storage/ninefold.rb +74 -0
  66. data/lib/backup/storage/rsync.rb +248 -0
  67. data/lib/backup/storage/s3.rb +154 -0
  68. data/lib/backup/storage/scp.rb +67 -0
  69. data/lib/backup/storage/sftp.rb +82 -0
  70. data/lib/backup/syncer/base.rb +70 -0
  71. data/lib/backup/syncer/cloud/base.rb +179 -0
  72. data/lib/backup/syncer/cloud/cloud_files.rb +83 -0
  73. data/lib/backup/syncer/cloud/local_file.rb +100 -0
  74. data/lib/backup/syncer/cloud/s3.rb +110 -0
  75. data/lib/backup/syncer/rsync/base.rb +48 -0
  76. data/lib/backup/syncer/rsync/local.rb +31 -0
  77. data/lib/backup/syncer/rsync/pull.rb +51 -0
  78. data/lib/backup/syncer/rsync/push.rb +205 -0
  79. data/lib/backup/template.rb +46 -0
  80. data/lib/backup/utilities.rb +224 -0
  81. data/lib/backup/version.rb +5 -0
  82. data/templates/cli/archive +28 -0
  83. data/templates/cli/compressor/bzip2 +4 -0
  84. data/templates/cli/compressor/custom +7 -0
  85. data/templates/cli/compressor/gzip +4 -0
  86. data/templates/cli/config +123 -0
  87. data/templates/cli/databases/mongodb +15 -0
  88. data/templates/cli/databases/mysql +18 -0
  89. data/templates/cli/databases/openldap +24 -0
  90. data/templates/cli/databases/postgresql +16 -0
  91. data/templates/cli/databases/redis +16 -0
  92. data/templates/cli/databases/riak +17 -0
  93. data/templates/cli/databases/sqlite +12 -0
  94. data/templates/cli/encryptor/gpg +27 -0
  95. data/templates/cli/encryptor/openssl +9 -0
  96. data/templates/cli/model +26 -0
  97. data/templates/cli/notifier/zabbix +15 -0
  98. data/templates/cli/notifiers/campfire +12 -0
  99. data/templates/cli/notifiers/datadog +57 -0
  100. data/templates/cli/notifiers/flowdock +16 -0
  101. data/templates/cli/notifiers/hipchat +15 -0
  102. data/templates/cli/notifiers/http_post +32 -0
  103. data/templates/cli/notifiers/mail +21 -0
  104. data/templates/cli/notifiers/nagios +13 -0
  105. data/templates/cli/notifiers/pagerduty +12 -0
  106. data/templates/cli/notifiers/prowl +11 -0
  107. data/templates/cli/notifiers/pushover +11 -0
  108. data/templates/cli/notifiers/slack +23 -0
  109. data/templates/cli/notifiers/twitter +13 -0
  110. data/templates/cli/splitter +7 -0
  111. data/templates/cli/storages/cloud_files +11 -0
  112. data/templates/cli/storages/dropbox +19 -0
  113. data/templates/cli/storages/ftp +12 -0
  114. data/templates/cli/storages/local +7 -0
  115. data/templates/cli/storages/ninefold +9 -0
  116. data/templates/cli/storages/rsync +17 -0
  117. data/templates/cli/storages/s3 +14 -0
  118. data/templates/cli/storages/scp +14 -0
  119. data/templates/cli/storages/sftp +14 -0
  120. data/templates/cli/syncers/cloud_files +22 -0
  121. data/templates/cli/syncers/rsync_local +20 -0
  122. data/templates/cli/syncers/rsync_pull +28 -0
  123. data/templates/cli/syncers/rsync_push +28 -0
  124. data/templates/cli/syncers/s3 +27 -0
  125. data/templates/general/links +3 -0
  126. data/templates/general/version.erb +2 -0
  127. data/templates/notifier/mail/failure.erb +16 -0
  128. data/templates/notifier/mail/success.erb +16 -0
  129. data/templates/notifier/mail/warning.erb +16 -0
  130. data/templates/storage/dropbox/authorization_url.erb +6 -0
  131. data/templates/storage/dropbox/authorized.erb +4 -0
  132. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  133. data/templates/storage/google_drive/authorization_url.erb +6 -0
  134. data/templates/storage/google_drive/authorized.erb +4 -0
  135. data/templates/storage/google_drive/cache_file_written.erb +10 -0
  136. metadata +957 -0
@@ -0,0 +1,98 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Notifier
5
+ class Error < Backup::Error; end
6
+
7
+ class Base
8
+ include Utilities::Helpers
9
+ include Config::Helpers
10
+
11
+ ##
12
+ # When set to true, the user will be notified by email
13
+ # when a backup process ends without raising any exceptions
14
+ attr_accessor :on_success
15
+ alias :notify_on_success? :on_success
16
+
17
+ ##
18
+ # When set to true, the user will be notified by email
19
+ # when a backup process is successful, but has warnings
20
+ attr_accessor :on_warning
21
+ alias :notify_on_warning? :on_warning
22
+
23
+ ##
24
+ # When set to true, the user will be notified by email
25
+ # when a backup process raises an exception before finishing
26
+ attr_accessor :on_failure
27
+ alias :notify_on_failure? :on_failure
28
+
29
+ ##
30
+ # Number of times to retry failed attempts to send notification.
31
+ # Default: 10
32
+ attr_accessor :max_retries
33
+
34
+ ##
35
+ # Time in seconds to pause before each retry.
36
+ # Default: 30
37
+ attr_accessor :retry_waitsec
38
+
39
+ attr_reader :model
40
+
41
+ def initialize(model)
42
+ @model = model
43
+ load_defaults!
44
+
45
+ @on_success = true if on_success.nil?
46
+ @on_warning = true if on_warning.nil?
47
+ @on_failure = true if on_failure.nil?
48
+ @max_retries ||= 10
49
+ @retry_waitsec ||= 30
50
+ end
51
+
52
+ # This method is called from an ensure block in Model#perform! and must
53
+ # not raise any exceptions. However, each Notifier's #notify! method
54
+ # should raise an exception if the request fails so it may be retried.
55
+ def perform!
56
+ status = case model.exit_status
57
+ when 0
58
+ :success if notify_on_success?
59
+ when 1
60
+ :warning if notify_on_success? || notify_on_warning?
61
+ else
62
+ :failure if notify_on_failure?
63
+ end
64
+
65
+ if status
66
+ Logger.info "Sending notification using #{ notifier_name }..."
67
+ with_retries { notify!(status) }
68
+ end
69
+
70
+ rescue Exception => err
71
+ Logger.error Error.wrap(err, "#{ notifier_name } Failed!")
72
+ end
73
+
74
+ private
75
+
76
+ def with_retries
77
+ retries = 0
78
+ begin
79
+ yield
80
+ rescue StandardError, Timeout::Error => err
81
+ retries += 1
82
+ raise if retries > max_retries
83
+
84
+ Logger.info Error.wrap(err, "Retry ##{ retries } of #{ max_retries }.")
85
+ sleep(retry_waitsec)
86
+ retry
87
+ end
88
+ end
89
+
90
+ ##
91
+ # Return the notifier name, with Backup namespace removed
92
+ def notifier_name
93
+ self.class.to_s.sub('Backup::', '')
94
+ end
95
+
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,69 @@
1
+ # encoding: utf-8
2
+ require 'json'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class Campfire < Base
7
+
8
+ ##
9
+ # Campfire api authentication token
10
+ attr_accessor :api_token
11
+
12
+ ##
13
+ # Campfire account's subdomain
14
+ attr_accessor :subdomain
15
+
16
+ ##
17
+ # Campfire account's room id
18
+ attr_accessor :room_id
19
+
20
+ def initialize(model, &block)
21
+ super
22
+ instance_eval(&block) if block_given?
23
+ end
24
+
25
+ private
26
+
27
+ ##
28
+ # Notify the user of the backup operation results.
29
+ #
30
+ # `status` indicates one of the following:
31
+ #
32
+ # `:success`
33
+ # : The backup completed successfully.
34
+ # : Notification will be sent if `on_success` is `true`.
35
+ #
36
+ # `:warning`
37
+ # : The backup completed successfully, but warnings were logged.
38
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
39
+ #
40
+ # `:failure`
41
+ # : The backup operation failed.
42
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
43
+ #
44
+ def notify!(status)
45
+ tag = case status
46
+ when :success then '[Backup::Success]'
47
+ when :warning then '[Backup::Warning]'
48
+ when :failure then '[Backup::Failure]'
49
+ end
50
+ message = "#{ tag } #{ model.label } (#{ model.trigger })"
51
+ send_message(message)
52
+ end
53
+
54
+ def send_message(message)
55
+ uri = "https://#{ subdomain }.campfirenow.com/room/#{ room_id }/speak.json"
56
+ options = {
57
+ :headers => { 'Content-Type' => 'application/json' },
58
+ :body => JSON.dump(
59
+ { :message => { :body => message, :type => 'Textmessage' } }
60
+ )
61
+ }
62
+ options.merge!(:user => api_token, :password => 'x') # Basic Auth
63
+ options.merge!(:expects => 201) # raise error if unsuccessful
64
+ Excon.post(uri, options)
65
+ end
66
+
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,116 @@
1
+ # encoding: utf-8
2
+ require 'dogapi'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class DataDog < Base
7
+
8
+ ##
9
+ # The DataDog API key
10
+ attr_accessor :api_key
11
+
12
+ ##
13
+ # The title of the event
14
+ attr_accessor :title
15
+
16
+ ##
17
+ # The text information for the event
18
+ attr_accessor :text
19
+
20
+ ##
21
+ # The timestamp for the event
22
+ attr_accessor :date_happened
23
+
24
+ ##
25
+ # The priority of the event (low/normal)
26
+ attr_accessor :priority
27
+
28
+ ##
29
+ # The host that generated the event
30
+ attr_accessor :host
31
+
32
+ ##
33
+ # The tags for this host (should be an array)
34
+ attr_accessor :tags
35
+
36
+ ##
37
+ # The alert_type of the event (error/warning/info/success)
38
+ attr_accessor :alert_type
39
+
40
+ ##
41
+ # The aggregation_key for the event
42
+ attr_accessor :aggregation_key
43
+
44
+ ##
45
+ # The source_type for the event (nagios, hudson, jenkins, user, my apps, feed, chef, puppet, git, bitbucket, fabric, capistrano)
46
+ attr_accessor :source_type_name
47
+
48
+ def initialize(model, &block)
49
+ super
50
+ instance_eval(&block) if block_given?
51
+
52
+ @title ||= default_title
53
+ @text ||= default_text
54
+ end
55
+
56
+ private
57
+
58
+ ##
59
+ # Notify the user of the backup operation results.
60
+ #
61
+ # `status` indicates one of the following:
62
+ #
63
+ # `:success`
64
+ # : The backup completed successfully.
65
+ # : Notification will be sent if `on_success` is `true`.
66
+ #
67
+ # `:warning`
68
+ # : The backup completed successfully, but warnings were logged.
69
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
70
+ #
71
+ # `:failure`
72
+ # : The backup operation failed.
73
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
74
+ #
75
+ def notify!(status)
76
+ hash = {alert_type: default_alert_type(status)}
77
+ hash.store(:msg_title, @title)
78
+ hash.store(:date_happened, @date_happened) if @date_happened
79
+ hash.store(:priority, @priority) if @priority
80
+ hash.store(:host, @host) if @host
81
+ hash.store(:tags, @tags) if @tags
82
+ hash.store(:aggregation_key, @aggregation_key) if @aggregation_key
83
+ hash.store(:source_type_name, @source_type_name) if @source_type_name
84
+ hash.store(:alert_type, @alert_type) if @alert_type
85
+ send_event(hash)
86
+ end
87
+
88
+ # Dogapi::Client will raise an error if unsuccessful.
89
+ def send_event(hash)
90
+ client = Dogapi::Client.new(@api_key)
91
+ event = Dogapi::Event.new(@text, hash)
92
+ client.emit_event(event)
93
+ end
94
+
95
+ # set alert type
96
+ def default_alert_type(status)
97
+ case status
98
+ when :success then 'success'
99
+ when :warning then 'warning'
100
+ when :failure then 'error'
101
+ end
102
+ end
103
+
104
+ # set default title
105
+ def default_title
106
+ "Backup #{ model.label }"
107
+ end
108
+
109
+ # set default text
110
+ def default_text
111
+ "Backup Notification for #{ model.label }"
112
+ end
113
+
114
+ end
115
+ end
116
+ end
@@ -0,0 +1,102 @@
1
+ # encoding: utf-8
2
+ require 'flowdock'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class FlowDock < Base
7
+
8
+ ##
9
+ # The Flowdock API token
10
+ attr_accessor :token
11
+
12
+ ##
13
+ # Who the notification should appear from
14
+ attr_accessor :from_name
15
+
16
+ # Which email the notification should appear from
17
+ attr_accessor :from_email
18
+
19
+ ##
20
+ # source for message
21
+ attr_accessor :source
22
+
23
+ ##
24
+ # Subject for message
25
+ attr_accessor :subject
26
+
27
+ ##
28
+ # tag message in inbox
29
+ attr_accessor :tags
30
+
31
+ ##
32
+ # link for message
33
+ attr_accessor :link
34
+
35
+ def initialize(model, &block)
36
+ super
37
+ instance_eval(&block) if block_given?
38
+
39
+ @subject ||= default_subject
40
+ @source ||= default_source
41
+ @tags ||= []
42
+ end
43
+
44
+ private
45
+
46
+ ##
47
+ # Notify the user of the backup operation results.
48
+ #
49
+ # `status` indicates one of the following:
50
+ #
51
+ # `:success`
52
+ # : The backup completed successfully.
53
+ # : Notification will be sent if `on_success` is `true`.
54
+ #
55
+ # `:warning`
56
+ # : The backup completed successfully, but warnings were logged.
57
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
58
+ #
59
+ # `:failure`
60
+ # : The backup operation failed.
61
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
62
+ #
63
+ def notify!(status)
64
+ @tags += default_tags(status)
65
+ message = "#{ model.label } (#{ model.trigger })"
66
+ send_message(message)
67
+ end
68
+
69
+ # Flowdock::Client will raise an error if unsuccessful.
70
+ def send_message(msg)
71
+ client = Flowdock::Flow.new(:api_token => token, :source => source,
72
+ :from => {:name => from_name, :address => from_email })
73
+
74
+ client.push_to_team_inbox(:subject => subject,
75
+ :content => msg,
76
+ :tags => tags,
77
+ :link => link )
78
+ end
79
+
80
+ # set related tags
81
+ def default_tags(status)
82
+ case status
83
+ when :success then ['#BackupSuccess']
84
+ when :warning then ['#BackupWarning']
85
+ when :failure then ['#BackupFailure']
86
+ end
87
+ end
88
+
89
+
90
+ #set default source
91
+ def default_source
92
+ "Backup #{ model.label }"
93
+ end
94
+
95
+ # set default subject
96
+ def default_subject
97
+ 'Backup Notification'
98
+ end
99
+
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,93 @@
1
+ # encoding: utf-8
2
+ require 'hipchat'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class Hipchat < Base
7
+
8
+ ##
9
+ # The Hipchat API token
10
+ attr_accessor :token
11
+
12
+ ##
13
+ # Who the notification should appear from
14
+ attr_accessor :from
15
+
16
+ ##
17
+ # The rooms that should be notified
18
+ attr_accessor :rooms_notified
19
+
20
+ ##
21
+ # Notify users in the room
22
+ attr_accessor :notify_users
23
+
24
+ ##
25
+ # The background color of a success message.
26
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
27
+ attr_accessor :success_color
28
+
29
+ ##
30
+ # The background color of a warning message.
31
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
32
+ attr_accessor :warning_color
33
+
34
+ ##
35
+ # The background color of an error message.
36
+ # One of :yellow, :red, :green, :purple, or :random. (default: yellow)
37
+ attr_accessor :failure_color
38
+
39
+ def initialize(model, &block)
40
+ super
41
+ instance_eval(&block) if block_given?
42
+
43
+ @notify_users ||= false
44
+ @rooms_notified ||= []
45
+ @success_color ||= 'yellow'
46
+ @warning_color ||= 'yellow'
47
+ @failure_color ||= 'yellow'
48
+ end
49
+
50
+ private
51
+
52
+ ##
53
+ # Notify the user of the backup operation results.
54
+ #
55
+ # `status` indicates one of the following:
56
+ #
57
+ # `:success`
58
+ # : The backup completed successfully.
59
+ # : Notification will be sent if `on_success` is `true`.
60
+ #
61
+ # `:warning`
62
+ # : The backup completed successfully, but warnings were logged.
63
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
64
+ #
65
+ # `:failure`
66
+ # : The backup operation failed.
67
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
68
+ #
69
+ def notify!(status)
70
+ tag, color = case status
71
+ when :success then ['[Backup::Success]', success_color]
72
+ when :warning then ['[Backup::Warning]', warning_color]
73
+ when :failure then ['[Backup::Failure]', failure_color]
74
+ end
75
+ message = "#{ tag } #{ model.label } (#{ model.trigger })"
76
+ send_message(message, color)
77
+ end
78
+
79
+ # Hipchat::Client will raise an error if unsuccessful.
80
+ def send_message(msg, color)
81
+ client = HipChat::Client.new(token)
82
+ rooms_to_notify.each do |room|
83
+ client[room].send(from, msg, :color => color, :notify => notify_users)
84
+ end
85
+ end
86
+
87
+ def rooms_to_notify
88
+ Array(rooms_notified).map {|r| r.split(',').map(&:strip) }.flatten
89
+ end
90
+
91
+ end
92
+ end
93
+ end