venet-backup 4.1.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (136) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.md +24 -0
  3. data/README.md +15 -0
  4. data/bin/backup +5 -0
  5. data/lib/backup.rb +141 -0
  6. data/lib/backup/archive.rb +170 -0
  7. data/lib/backup/binder.rb +22 -0
  8. data/lib/backup/cleaner.rb +116 -0
  9. data/lib/backup/cli.rb +374 -0
  10. data/lib/backup/cloud_io/base.rb +41 -0
  11. data/lib/backup/cloud_io/cloud_files.rb +298 -0
  12. data/lib/backup/cloud_io/s3.rb +260 -0
  13. data/lib/backup/compressor/base.rb +35 -0
  14. data/lib/backup/compressor/bzip2.rb +39 -0
  15. data/lib/backup/compressor/custom.rb +53 -0
  16. data/lib/backup/compressor/gzip.rb +74 -0
  17. data/lib/backup/config.rb +119 -0
  18. data/lib/backup/config/dsl.rb +103 -0
  19. data/lib/backup/config/helpers.rb +143 -0
  20. data/lib/backup/database/base.rb +85 -0
  21. data/lib/backup/database/mongodb.rb +186 -0
  22. data/lib/backup/database/mysql.rb +180 -0
  23. data/lib/backup/database/openldap.rb +95 -0
  24. data/lib/backup/database/postgresql.rb +133 -0
  25. data/lib/backup/database/redis.rb +179 -0
  26. data/lib/backup/database/riak.rb +82 -0
  27. data/lib/backup/database/sqlite.rb +57 -0
  28. data/lib/backup/encryptor/base.rb +29 -0
  29. data/lib/backup/encryptor/gpg.rb +747 -0
  30. data/lib/backup/encryptor/open_ssl.rb +72 -0
  31. data/lib/backup/errors.rb +58 -0
  32. data/lib/backup/logger.rb +199 -0
  33. data/lib/backup/logger/console.rb +51 -0
  34. data/lib/backup/logger/fog_adapter.rb +29 -0
  35. data/lib/backup/logger/logfile.rb +133 -0
  36. data/lib/backup/logger/syslog.rb +116 -0
  37. data/lib/backup/model.rb +454 -0
  38. data/lib/backup/notifier/base.rb +98 -0
  39. data/lib/backup/notifier/campfire.rb +69 -0
  40. data/lib/backup/notifier/datadog.rb +116 -0
  41. data/lib/backup/notifier/flowdock.rb +102 -0
  42. data/lib/backup/notifier/hipchat.rb +93 -0
  43. data/lib/backup/notifier/http_post.rb +122 -0
  44. data/lib/backup/notifier/mail.rb +238 -0
  45. data/lib/backup/notifier/nagios.rb +74 -0
  46. data/lib/backup/notifier/pagerduty.rb +81 -0
  47. data/lib/backup/notifier/prowl.rb +69 -0
  48. data/lib/backup/notifier/pushover.rb +80 -0
  49. data/lib/backup/notifier/slack.rb +158 -0
  50. data/lib/backup/notifier/twitter.rb +64 -0
  51. data/lib/backup/notifier/zabbix.rb +68 -0
  52. data/lib/backup/package.rb +51 -0
  53. data/lib/backup/packager.rb +101 -0
  54. data/lib/backup/pipeline.rb +124 -0
  55. data/lib/backup/splitter.rb +76 -0
  56. data/lib/backup/storage/base.rb +57 -0
  57. data/lib/backup/storage/cloud_files.rb +158 -0
  58. data/lib/backup/storage/cycler.rb +65 -0
  59. data/lib/backup/storage/dropbox.rb +236 -0
  60. data/lib/backup/storage/ftp.rb +98 -0
  61. data/lib/backup/storage/google/google_drive_auth.rb +96 -0
  62. data/lib/backup/storage/google/google_drive_transfer.rb +125 -0
  63. data/lib/backup/storage/google_drive.rb +62 -0
  64. data/lib/backup/storage/local.rb +64 -0
  65. data/lib/backup/storage/ninefold.rb +74 -0
  66. data/lib/backup/storage/rsync.rb +248 -0
  67. data/lib/backup/storage/s3.rb +154 -0
  68. data/lib/backup/storage/scp.rb +67 -0
  69. data/lib/backup/storage/sftp.rb +82 -0
  70. data/lib/backup/syncer/base.rb +70 -0
  71. data/lib/backup/syncer/cloud/base.rb +179 -0
  72. data/lib/backup/syncer/cloud/cloud_files.rb +83 -0
  73. data/lib/backup/syncer/cloud/local_file.rb +100 -0
  74. data/lib/backup/syncer/cloud/s3.rb +110 -0
  75. data/lib/backup/syncer/rsync/base.rb +48 -0
  76. data/lib/backup/syncer/rsync/local.rb +31 -0
  77. data/lib/backup/syncer/rsync/pull.rb +51 -0
  78. data/lib/backup/syncer/rsync/push.rb +205 -0
  79. data/lib/backup/template.rb +46 -0
  80. data/lib/backup/utilities.rb +224 -0
  81. data/lib/backup/version.rb +5 -0
  82. data/templates/cli/archive +28 -0
  83. data/templates/cli/compressor/bzip2 +4 -0
  84. data/templates/cli/compressor/custom +7 -0
  85. data/templates/cli/compressor/gzip +4 -0
  86. data/templates/cli/config +123 -0
  87. data/templates/cli/databases/mongodb +15 -0
  88. data/templates/cli/databases/mysql +18 -0
  89. data/templates/cli/databases/openldap +24 -0
  90. data/templates/cli/databases/postgresql +16 -0
  91. data/templates/cli/databases/redis +16 -0
  92. data/templates/cli/databases/riak +17 -0
  93. data/templates/cli/databases/sqlite +12 -0
  94. data/templates/cli/encryptor/gpg +27 -0
  95. data/templates/cli/encryptor/openssl +9 -0
  96. data/templates/cli/model +26 -0
  97. data/templates/cli/notifier/zabbix +15 -0
  98. data/templates/cli/notifiers/campfire +12 -0
  99. data/templates/cli/notifiers/datadog +57 -0
  100. data/templates/cli/notifiers/flowdock +16 -0
  101. data/templates/cli/notifiers/hipchat +15 -0
  102. data/templates/cli/notifiers/http_post +32 -0
  103. data/templates/cli/notifiers/mail +21 -0
  104. data/templates/cli/notifiers/nagios +13 -0
  105. data/templates/cli/notifiers/pagerduty +12 -0
  106. data/templates/cli/notifiers/prowl +11 -0
  107. data/templates/cli/notifiers/pushover +11 -0
  108. data/templates/cli/notifiers/slack +23 -0
  109. data/templates/cli/notifiers/twitter +13 -0
  110. data/templates/cli/splitter +7 -0
  111. data/templates/cli/storages/cloud_files +11 -0
  112. data/templates/cli/storages/dropbox +19 -0
  113. data/templates/cli/storages/ftp +12 -0
  114. data/templates/cli/storages/local +7 -0
  115. data/templates/cli/storages/ninefold +9 -0
  116. data/templates/cli/storages/rsync +17 -0
  117. data/templates/cli/storages/s3 +14 -0
  118. data/templates/cli/storages/scp +14 -0
  119. data/templates/cli/storages/sftp +14 -0
  120. data/templates/cli/syncers/cloud_files +22 -0
  121. data/templates/cli/syncers/rsync_local +20 -0
  122. data/templates/cli/syncers/rsync_pull +28 -0
  123. data/templates/cli/syncers/rsync_push +28 -0
  124. data/templates/cli/syncers/s3 +27 -0
  125. data/templates/general/links +3 -0
  126. data/templates/general/version.erb +2 -0
  127. data/templates/notifier/mail/failure.erb +16 -0
  128. data/templates/notifier/mail/success.erb +16 -0
  129. data/templates/notifier/mail/warning.erb +16 -0
  130. data/templates/storage/dropbox/authorization_url.erb +6 -0
  131. data/templates/storage/dropbox/authorized.erb +4 -0
  132. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  133. data/templates/storage/google_drive/authorization_url.erb +6 -0
  134. data/templates/storage/google_drive/authorized.erb +4 -0
  135. data/templates/storage/google_drive/cache_file_written.erb +10 -0
  136. metadata +957 -0
@@ -0,0 +1,69 @@
1
+ # encoding: utf-8
2
+ require 'uri'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class Prowl < Base
7
+
8
+ ##
9
+ # Application name
10
+ # Tell something like your server name. Example: "Server1 Backup"
11
+ attr_accessor :application
12
+
13
+ ##
14
+ # API-Key
15
+ # Create a Prowl account and request an API key on prowlapp.com.
16
+ attr_accessor :api_key
17
+
18
+ def initialize(model, &block)
19
+ super
20
+ instance_eval(&block) if block_given?
21
+ end
22
+
23
+ private
24
+
25
+ ##
26
+ # Notify the user of the backup operation results.
27
+ #
28
+ # `status` indicates one of the following:
29
+ #
30
+ # `:success`
31
+ # : The backup completed successfully.
32
+ # : Notification will be sent if `on_success` is `true`.
33
+ #
34
+ # `:warning`
35
+ # : The backup completed successfully, but warnings were logged.
36
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
37
+ #
38
+ # `:failure`
39
+ # : The backup operation failed.
40
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
41
+ #
42
+ def notify!(status)
43
+ tag = case status
44
+ when :success then '[Backup::Success]'
45
+ when :warning then '[Backup::Warning]'
46
+ when :failure then '[Backup::Failure]'
47
+ end
48
+ send_message(tag)
49
+ end
50
+
51
+ def send_message(message)
52
+ uri = 'https://api.prowlapp.com/publicapi/add'
53
+ data = {
54
+ :application => application,
55
+ :apikey => api_key,
56
+ :event => message,
57
+ :description => "#{ model.label } (#{ model.trigger })"
58
+ }
59
+ options = {
60
+ :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
61
+ :body => URI.encode_www_form(data)
62
+ }
63
+ options.merge!(:expects => 200) # raise error if unsuccessful
64
+ Excon.post(uri, options)
65
+ end
66
+
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,80 @@
1
+ # encoding: utf-8
2
+ require 'uri'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class Pushover < Base
7
+
8
+ ##
9
+ # The API User Token
10
+ attr_accessor :user
11
+
12
+ ##
13
+ # The API Application Token
14
+ attr_accessor :token
15
+
16
+ ##
17
+ # The user's device identifier to sent the message directly to,
18
+ # rather than all of the user's devices
19
+ attr_accessor :device
20
+
21
+ ##
22
+ # The message title
23
+ attr_accessor :title
24
+
25
+ ##
26
+ # The priority of the notification
27
+ attr_accessor :priority
28
+
29
+ def initialize(model, &block)
30
+ super
31
+ instance_eval(&block) if block_given?
32
+ end
33
+
34
+ private
35
+
36
+ ##
37
+ # Notify the user of the backup operation results.
38
+ #
39
+ # `status` indicates one of the following:
40
+ #
41
+ # `:success`
42
+ # : The backup completed successfully.
43
+ # : Notification will be sent if `on_success` is `true`.
44
+ #
45
+ # `:warning`
46
+ # : The backup completed successfully, but warnings were logged.
47
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
48
+ #
49
+ # `:failure`
50
+ # : The backup operation failed.
51
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
52
+ #
53
+ def notify!(status)
54
+ tag = case status
55
+ when :success then '[Backup::Success]'
56
+ when :failure then '[Backup::Failure]'
57
+ when :warning then '[Backup::Warning]'
58
+ end
59
+ message = "#{ tag } #{ model.label } (#{ model.trigger })"
60
+ send_message(message)
61
+ end
62
+
63
+ def send_message(message)
64
+ uri = 'https://api.pushover.net/1/messages.json'
65
+ data = { :user => user, :token => token, :message => message }
66
+ [:device, :title, :priority].each do |param|
67
+ val = send(param)
68
+ data.merge!(param => val) if val
69
+ end
70
+ options = {
71
+ :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
72
+ :body => URI.encode_www_form(data)
73
+ }
74
+ options.merge!(:expects => 200) # raise error if unsuccessful
75
+ Excon.post(uri, options)
76
+ end
77
+
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,158 @@
1
+ # encoding: utf-8
2
+ require 'uri'
3
+ require 'json'
4
+
5
+ module Backup
6
+ module Notifier
7
+ class Slack < Base
8
+
9
+ ##
10
+ # The Team name
11
+ attr_accessor :team
12
+
13
+ ##
14
+ # The Integration Token
15
+ attr_accessor :token
16
+
17
+ ##
18
+ # The channel to send messages to
19
+ attr_accessor :channel
20
+
21
+ ##
22
+ # The username to display along with the notification
23
+ attr_accessor :username
24
+
25
+ ##
26
+ # The emoji icon to display along with the notification
27
+ #
28
+ # See http://www.emoji-cheat-sheet.com for a list of icons.
29
+ #
30
+ # Default: :floppy_disk:
31
+ attr_accessor :icon_emoji
32
+
33
+ ##
34
+ # Array of statuses for which the log file should be attached.
35
+ #
36
+ # Available statuses are: `:success`, `:warning` and `:failure`.
37
+ # Default: [:warning, :failure]
38
+ attr_accessor :send_log_on
39
+
40
+ def initialize(model, &block)
41
+ super
42
+ instance_eval(&block) if block_given?
43
+
44
+ @send_log_on ||= [:warning, :failure]
45
+ @icon_emoji ||= ':floppy_disk:'
46
+ end
47
+
48
+ private
49
+
50
+ ##
51
+ # Notify the user of the backup operation results.
52
+ #
53
+ # `status` indicates one of the following:
54
+ #
55
+ # `:success`
56
+ # : The backup completed successfully.
57
+ # : Notification will be sent if `on_success` is `true`.
58
+ #
59
+ # `:warning`
60
+ # : The backup completed successfully, but warnings were logged.
61
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
62
+ #
63
+ # `:failure`
64
+ # : The backup operation failed.
65
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
66
+ #
67
+ def notify!(status)
68
+ tag = case status
69
+ when :success then '[Backup::Success]'
70
+ when :failure then '[Backup::Failure]'
71
+ when :warning then '[Backup::Warning]'
72
+ end
73
+ message = "#{ tag } #{ model.label } (#{ model.trigger })"
74
+
75
+ data = { :text => message }
76
+ [:channel, :username, :icon_emoji].each do |param|
77
+ val = send(param)
78
+ data.merge!(param => val) if val
79
+ end
80
+
81
+ data.merge!(:attachments => [attachment(status)])
82
+
83
+ options = {
84
+ :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
85
+ :body => URI.encode_www_form(:payload => JSON.dump(data))
86
+ }
87
+ options.merge!(:expects => 200) # raise error if unsuccessful
88
+ Excon.post(uri, options)
89
+ end
90
+
91
+ def attachment(status)
92
+ {
93
+ :fallback => "#{title(status)} - Job: #{model.label} (#{model.trigger})",
94
+ :text => title(status),
95
+ :color => color(status),
96
+ :fields => [
97
+ {
98
+ :title => "Job",
99
+ :value => "#{model.label} (#{model.trigger})",
100
+ :short => false
101
+ },
102
+ {
103
+ :title => "Started",
104
+ :value => model.started_at,
105
+ :short => true
106
+ },
107
+ {
108
+ :title => "Finished",
109
+ :value => model.finished_at,
110
+ :short => true
111
+ },
112
+ {
113
+ :title => "Duration",
114
+ :value => model.duration,
115
+ :short => true
116
+ },
117
+ {
118
+ :title => "Version",
119
+ :value => "Backup v#{Backup::VERSION}\nRuby: #{RUBY_DESCRIPTION}",
120
+ :short => false
121
+ },
122
+ log_field(status)
123
+ ].compact
124
+ }
125
+ end
126
+
127
+ def log_field(status)
128
+ send_log = send_log_on.include?(status)
129
+
130
+ return {
131
+ :title => "Detailed Backup Log",
132
+ :value => Logger.messages.map(&:formatted_lines).flatten.join("\n"),
133
+ :short => false,
134
+ } if send_log
135
+ end
136
+
137
+ def color(status)
138
+ case status
139
+ when :success then 'good'
140
+ when :failure then 'danger'
141
+ when :warning then 'warning'
142
+ end
143
+ end
144
+
145
+ def title(status)
146
+ case status
147
+ when :success then 'Backup Completed Successfully!'
148
+ when :failure then 'Backup Failed!'
149
+ when :warning then 'Backup Completed Successfully (with Warnings)!'
150
+ end
151
+ end
152
+
153
+ def uri
154
+ @uri ||= "https://#{team}.slack.com/services/hooks/incoming-webhook?token=#{token}"
155
+ end
156
+ end
157
+ end
158
+ end
@@ -0,0 +1,64 @@
1
+ # encoding: utf-8
2
+ require 'twitter'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class Twitter < Base
7
+
8
+ ##
9
+ # Twitter consumer key credentials
10
+ attr_accessor :consumer_key, :consumer_secret
11
+
12
+ ##
13
+ # OAuth credentials
14
+ attr_accessor :oauth_token, :oauth_token_secret
15
+
16
+ def initialize(model, &block)
17
+ super
18
+ instance_eval(&block) if block_given?
19
+ end
20
+
21
+ private
22
+
23
+ ##
24
+ # Notify the user of the backup operation results.
25
+ #
26
+ # `status` indicates one of the following:
27
+ #
28
+ # `:success`
29
+ # : The backup completed successfully.
30
+ # : Notification will be sent if `on_success` is `true`.
31
+ #
32
+ # `:warning`
33
+ # : The backup completed successfully, but warnings were logged.
34
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
35
+ #
36
+ # `:failure`
37
+ # : The backup operation failed.
38
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
39
+ #
40
+ def notify!(status)
41
+ tag = case status
42
+ when :success then '[Backup::Success]'
43
+ when :warning then '[Backup::Warning]'
44
+ when :failure then '[Backup::Failure]'
45
+ end
46
+ message = "#{ tag } #{ model.label } (#{ model.trigger }) (@ #{ model.time })"
47
+ send_message(message)
48
+ end
49
+
50
+ # Twitter::Client will raise an error if unsuccessful.
51
+ def send_message(message)
52
+ client = ::Twitter::REST::Client.new do |config|
53
+ config.consumer_key = @consumer_key
54
+ config.consumer_secret = @consumer_secret
55
+ config.access_token = @oauth_token
56
+ config.access_token_secret = @oauth_token_secret
57
+ end
58
+
59
+ client.update(message)
60
+ end
61
+
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,68 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Notifier
5
+ class Zabbix < Base
6
+
7
+ attr_accessor :zabbix_host
8
+
9
+ attr_accessor :zabbix_port
10
+
11
+ attr_accessor :service_name
12
+
13
+ attr_accessor :service_host
14
+
15
+ attr_accessor :item_key
16
+
17
+ def initialize(model, &block)
18
+ super
19
+ instance_eval(&block) if block_given?
20
+
21
+ @zabbix_host ||= Config.hostname
22
+ @zabbix_port ||= 10051
23
+ @service_name ||= "Backup #{ model.trigger }"
24
+ @service_host ||= Config.hostname
25
+ @item_key ||= 'backup_status'
26
+ end
27
+
28
+ private
29
+
30
+ ##
31
+ # Notify the user of the backup operation results.
32
+ #
33
+ # `status` indicates one of the following:
34
+ #
35
+ # `:success`
36
+ # : The backup completed successfully.
37
+ # : Notification will be sent if `on_success` is `true`.
38
+ #
39
+ # `:warning`
40
+ # : The backup completed successfully, but warnings were logged.
41
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
42
+ #
43
+ # `:failure`
44
+ # : The backup operation failed.
45
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
46
+ #
47
+ def notify!(status)
48
+ message = case status
49
+ when :success then 'Completed Successfully'
50
+ when :warning then 'Completed Successfully (with Warnings)'
51
+ when :failure then 'Failed'
52
+ end
53
+ send_message("#{ message } in #{ model.duration }")
54
+ end
55
+
56
+ def send_message(message)
57
+ msg = [service_host, service_name, model.exit_status, message].join("\t")
58
+ cmd = "#{ utility(:zabbix_sender) }" +
59
+ " -z '#{ zabbix_host }'" +
60
+ " -p '#{ zabbix_port }'" +
61
+ " -s #{ service_host }" +
62
+ " -k #{ item_key }" +
63
+ " -o '#{ msg }'"
64
+ run("echo '#{ msg }' | #{ cmd }")
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,51 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ class Package
5
+
6
+ ##
7
+ # The time when the backup initiated (in format: 2011.02.20.03.29.59)
8
+ attr_accessor :time
9
+
10
+ ##
11
+ # The trigger which initiated the backup process
12
+ attr_reader :trigger
13
+
14
+ ##
15
+ # Extension for the final archive file(s)
16
+ attr_accessor :extension
17
+
18
+ ##
19
+ # Set by the Splitter if the final archive was "chunked"
20
+ attr_accessor :chunk_suffixes
21
+
22
+ ##
23
+ # If true, the Cycler will not attempt to remove the package when Cycling.
24
+ attr_accessor :no_cycle
25
+
26
+ ##
27
+ # The version of Backup used to create the package
28
+ attr_reader :version
29
+
30
+ def initialize(model)
31
+ @trigger = model.trigger
32
+ @extension = 'tar'
33
+ @chunk_suffixes = Array.new
34
+ @no_cycle = false
35
+ @version = VERSION
36
+ end
37
+
38
+ def filenames
39
+ if chunk_suffixes.empty?
40
+ [basename]
41
+ else
42
+ chunk_suffixes.map {|suffix| "#{ basename }-#{ suffix }" }
43
+ end
44
+ end
45
+
46
+ def basename
47
+ "#{ trigger }.#{ extension }"
48
+ end
49
+
50
+ end
51
+ end