venet-backup 4.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.md +24 -0
  3. data/README.md +15 -0
  4. data/bin/backup +5 -0
  5. data/lib/backup.rb +141 -0
  6. data/lib/backup/archive.rb +170 -0
  7. data/lib/backup/binder.rb +22 -0
  8. data/lib/backup/cleaner.rb +116 -0
  9. data/lib/backup/cli.rb +374 -0
  10. data/lib/backup/cloud_io/base.rb +41 -0
  11. data/lib/backup/cloud_io/cloud_files.rb +298 -0
  12. data/lib/backup/cloud_io/s3.rb +260 -0
  13. data/lib/backup/compressor/base.rb +35 -0
  14. data/lib/backup/compressor/bzip2.rb +39 -0
  15. data/lib/backup/compressor/custom.rb +53 -0
  16. data/lib/backup/compressor/gzip.rb +74 -0
  17. data/lib/backup/config.rb +119 -0
  18. data/lib/backup/config/dsl.rb +103 -0
  19. data/lib/backup/config/helpers.rb +143 -0
  20. data/lib/backup/database/base.rb +85 -0
  21. data/lib/backup/database/mongodb.rb +186 -0
  22. data/lib/backup/database/mysql.rb +180 -0
  23. data/lib/backup/database/openldap.rb +95 -0
  24. data/lib/backup/database/postgresql.rb +133 -0
  25. data/lib/backup/database/redis.rb +179 -0
  26. data/lib/backup/database/riak.rb +82 -0
  27. data/lib/backup/database/sqlite.rb +57 -0
  28. data/lib/backup/encryptor/base.rb +29 -0
  29. data/lib/backup/encryptor/gpg.rb +747 -0
  30. data/lib/backup/encryptor/open_ssl.rb +72 -0
  31. data/lib/backup/errors.rb +58 -0
  32. data/lib/backup/logger.rb +199 -0
  33. data/lib/backup/logger/console.rb +51 -0
  34. data/lib/backup/logger/fog_adapter.rb +29 -0
  35. data/lib/backup/logger/logfile.rb +133 -0
  36. data/lib/backup/logger/syslog.rb +116 -0
  37. data/lib/backup/model.rb +454 -0
  38. data/lib/backup/notifier/base.rb +98 -0
  39. data/lib/backup/notifier/campfire.rb +69 -0
  40. data/lib/backup/notifier/datadog.rb +116 -0
  41. data/lib/backup/notifier/flowdock.rb +102 -0
  42. data/lib/backup/notifier/hipchat.rb +93 -0
  43. data/lib/backup/notifier/http_post.rb +122 -0
  44. data/lib/backup/notifier/mail.rb +238 -0
  45. data/lib/backup/notifier/nagios.rb +74 -0
  46. data/lib/backup/notifier/pagerduty.rb +81 -0
  47. data/lib/backup/notifier/prowl.rb +69 -0
  48. data/lib/backup/notifier/pushover.rb +80 -0
  49. data/lib/backup/notifier/slack.rb +158 -0
  50. data/lib/backup/notifier/twitter.rb +64 -0
  51. data/lib/backup/notifier/zabbix.rb +68 -0
  52. data/lib/backup/package.rb +51 -0
  53. data/lib/backup/packager.rb +101 -0
  54. data/lib/backup/pipeline.rb +124 -0
  55. data/lib/backup/splitter.rb +76 -0
  56. data/lib/backup/storage/base.rb +57 -0
  57. data/lib/backup/storage/cloud_files.rb +158 -0
  58. data/lib/backup/storage/cycler.rb +65 -0
  59. data/lib/backup/storage/dropbox.rb +236 -0
  60. data/lib/backup/storage/ftp.rb +98 -0
  61. data/lib/backup/storage/google/google_drive_auth.rb +96 -0
  62. data/lib/backup/storage/google/google_drive_transfer.rb +125 -0
  63. data/lib/backup/storage/google_drive.rb +62 -0
  64. data/lib/backup/storage/local.rb +64 -0
  65. data/lib/backup/storage/ninefold.rb +74 -0
  66. data/lib/backup/storage/rsync.rb +248 -0
  67. data/lib/backup/storage/s3.rb +154 -0
  68. data/lib/backup/storage/scp.rb +67 -0
  69. data/lib/backup/storage/sftp.rb +82 -0
  70. data/lib/backup/syncer/base.rb +70 -0
  71. data/lib/backup/syncer/cloud/base.rb +179 -0
  72. data/lib/backup/syncer/cloud/cloud_files.rb +83 -0
  73. data/lib/backup/syncer/cloud/local_file.rb +100 -0
  74. data/lib/backup/syncer/cloud/s3.rb +110 -0
  75. data/lib/backup/syncer/rsync/base.rb +48 -0
  76. data/lib/backup/syncer/rsync/local.rb +31 -0
  77. data/lib/backup/syncer/rsync/pull.rb +51 -0
  78. data/lib/backup/syncer/rsync/push.rb +205 -0
  79. data/lib/backup/template.rb +46 -0
  80. data/lib/backup/utilities.rb +224 -0
  81. data/lib/backup/version.rb +5 -0
  82. data/templates/cli/archive +28 -0
  83. data/templates/cli/compressor/bzip2 +4 -0
  84. data/templates/cli/compressor/custom +7 -0
  85. data/templates/cli/compressor/gzip +4 -0
  86. data/templates/cli/config +123 -0
  87. data/templates/cli/databases/mongodb +15 -0
  88. data/templates/cli/databases/mysql +18 -0
  89. data/templates/cli/databases/openldap +24 -0
  90. data/templates/cli/databases/postgresql +16 -0
  91. data/templates/cli/databases/redis +16 -0
  92. data/templates/cli/databases/riak +17 -0
  93. data/templates/cli/databases/sqlite +12 -0
  94. data/templates/cli/encryptor/gpg +27 -0
  95. data/templates/cli/encryptor/openssl +9 -0
  96. data/templates/cli/model +26 -0
  97. data/templates/cli/notifier/zabbix +15 -0
  98. data/templates/cli/notifiers/campfire +12 -0
  99. data/templates/cli/notifiers/datadog +57 -0
  100. data/templates/cli/notifiers/flowdock +16 -0
  101. data/templates/cli/notifiers/hipchat +15 -0
  102. data/templates/cli/notifiers/http_post +32 -0
  103. data/templates/cli/notifiers/mail +21 -0
  104. data/templates/cli/notifiers/nagios +13 -0
  105. data/templates/cli/notifiers/pagerduty +12 -0
  106. data/templates/cli/notifiers/prowl +11 -0
  107. data/templates/cli/notifiers/pushover +11 -0
  108. data/templates/cli/notifiers/slack +23 -0
  109. data/templates/cli/notifiers/twitter +13 -0
  110. data/templates/cli/splitter +7 -0
  111. data/templates/cli/storages/cloud_files +11 -0
  112. data/templates/cli/storages/dropbox +19 -0
  113. data/templates/cli/storages/ftp +12 -0
  114. data/templates/cli/storages/local +7 -0
  115. data/templates/cli/storages/ninefold +9 -0
  116. data/templates/cli/storages/rsync +17 -0
  117. data/templates/cli/storages/s3 +14 -0
  118. data/templates/cli/storages/scp +14 -0
  119. data/templates/cli/storages/sftp +14 -0
  120. data/templates/cli/syncers/cloud_files +22 -0
  121. data/templates/cli/syncers/rsync_local +20 -0
  122. data/templates/cli/syncers/rsync_pull +28 -0
  123. data/templates/cli/syncers/rsync_push +28 -0
  124. data/templates/cli/syncers/s3 +27 -0
  125. data/templates/general/links +3 -0
  126. data/templates/general/version.erb +2 -0
  127. data/templates/notifier/mail/failure.erb +16 -0
  128. data/templates/notifier/mail/success.erb +16 -0
  129. data/templates/notifier/mail/warning.erb +16 -0
  130. data/templates/storage/dropbox/authorization_url.erb +6 -0
  131. data/templates/storage/dropbox/authorized.erb +4 -0
  132. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  133. data/templates/storage/google_drive/authorization_url.erb +6 -0
  134. data/templates/storage/google_drive/authorized.erb +4 -0
  135. data/templates/storage/google_drive/cache_file_written.erb +10 -0
  136. metadata +957 -0
@@ -0,0 +1,69 @@
1
+ # encoding: utf-8
2
+ require 'uri'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class Prowl < Base
7
+
8
+ ##
9
+ # Application name
10
+ # Tell something like your server name. Example: "Server1 Backup"
11
+ attr_accessor :application
12
+
13
+ ##
14
+ # API-Key
15
+ # Create a Prowl account and request an API key on prowlapp.com.
16
+ attr_accessor :api_key
17
+
18
+ def initialize(model, &block)
19
+ super
20
+ instance_eval(&block) if block_given?
21
+ end
22
+
23
+ private
24
+
25
+ ##
26
+ # Notify the user of the backup operation results.
27
+ #
28
+ # `status` indicates one of the following:
29
+ #
30
+ # `:success`
31
+ # : The backup completed successfully.
32
+ # : Notification will be sent if `on_success` is `true`.
33
+ #
34
+ # `:warning`
35
+ # : The backup completed successfully, but warnings were logged.
36
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
37
+ #
38
+ # `:failure`
39
+ # : The backup operation failed.
40
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
41
+ #
42
+ def notify!(status)
43
+ tag = case status
44
+ when :success then '[Backup::Success]'
45
+ when :warning then '[Backup::Warning]'
46
+ when :failure then '[Backup::Failure]'
47
+ end
48
+ send_message(tag)
49
+ end
50
+
51
+ def send_message(message)
52
+ uri = 'https://api.prowlapp.com/publicapi/add'
53
+ data = {
54
+ :application => application,
55
+ :apikey => api_key,
56
+ :event => message,
57
+ :description => "#{ model.label } (#{ model.trigger })"
58
+ }
59
+ options = {
60
+ :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
61
+ :body => URI.encode_www_form(data)
62
+ }
63
+ options.merge!(:expects => 200) # raise error if unsuccessful
64
+ Excon.post(uri, options)
65
+ end
66
+
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,80 @@
1
+ # encoding: utf-8
2
+ require 'uri'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class Pushover < Base
7
+
8
+ ##
9
+ # The API User Token
10
+ attr_accessor :user
11
+
12
+ ##
13
+ # The API Application Token
14
+ attr_accessor :token
15
+
16
+ ##
17
+ # The user's device identifier to sent the message directly to,
18
+ # rather than all of the user's devices
19
+ attr_accessor :device
20
+
21
+ ##
22
+ # The message title
23
+ attr_accessor :title
24
+
25
+ ##
26
+ # The priority of the notification
27
+ attr_accessor :priority
28
+
29
+ def initialize(model, &block)
30
+ super
31
+ instance_eval(&block) if block_given?
32
+ end
33
+
34
+ private
35
+
36
+ ##
37
+ # Notify the user of the backup operation results.
38
+ #
39
+ # `status` indicates one of the following:
40
+ #
41
+ # `:success`
42
+ # : The backup completed successfully.
43
+ # : Notification will be sent if `on_success` is `true`.
44
+ #
45
+ # `:warning`
46
+ # : The backup completed successfully, but warnings were logged.
47
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
48
+ #
49
+ # `:failure`
50
+ # : The backup operation failed.
51
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
52
+ #
53
+ def notify!(status)
54
+ tag = case status
55
+ when :success then '[Backup::Success]'
56
+ when :failure then '[Backup::Failure]'
57
+ when :warning then '[Backup::Warning]'
58
+ end
59
+ message = "#{ tag } #{ model.label } (#{ model.trigger })"
60
+ send_message(message)
61
+ end
62
+
63
+ def send_message(message)
64
+ uri = 'https://api.pushover.net/1/messages.json'
65
+ data = { :user => user, :token => token, :message => message }
66
+ [:device, :title, :priority].each do |param|
67
+ val = send(param)
68
+ data.merge!(param => val) if val
69
+ end
70
+ options = {
71
+ :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
72
+ :body => URI.encode_www_form(data)
73
+ }
74
+ options.merge!(:expects => 200) # raise error if unsuccessful
75
+ Excon.post(uri, options)
76
+ end
77
+
78
+ end
79
+ end
80
+ end
@@ -0,0 +1,158 @@
1
+ # encoding: utf-8
2
+ require 'uri'
3
+ require 'json'
4
+
5
+ module Backup
6
+ module Notifier
7
+ class Slack < Base
8
+
9
+ ##
10
+ # The Team name
11
+ attr_accessor :team
12
+
13
+ ##
14
+ # The Integration Token
15
+ attr_accessor :token
16
+
17
+ ##
18
+ # The channel to send messages to
19
+ attr_accessor :channel
20
+
21
+ ##
22
+ # The username to display along with the notification
23
+ attr_accessor :username
24
+
25
+ ##
26
+ # The emoji icon to display along with the notification
27
+ #
28
+ # See http://www.emoji-cheat-sheet.com for a list of icons.
29
+ #
30
+ # Default: :floppy_disk:
31
+ attr_accessor :icon_emoji
32
+
33
+ ##
34
+ # Array of statuses for which the log file should be attached.
35
+ #
36
+ # Available statuses are: `:success`, `:warning` and `:failure`.
37
+ # Default: [:warning, :failure]
38
+ attr_accessor :send_log_on
39
+
40
+ def initialize(model, &block)
41
+ super
42
+ instance_eval(&block) if block_given?
43
+
44
+ @send_log_on ||= [:warning, :failure]
45
+ @icon_emoji ||= ':floppy_disk:'
46
+ end
47
+
48
+ private
49
+
50
+ ##
51
+ # Notify the user of the backup operation results.
52
+ #
53
+ # `status` indicates one of the following:
54
+ #
55
+ # `:success`
56
+ # : The backup completed successfully.
57
+ # : Notification will be sent if `on_success` is `true`.
58
+ #
59
+ # `:warning`
60
+ # : The backup completed successfully, but warnings were logged.
61
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
62
+ #
63
+ # `:failure`
64
+ # : The backup operation failed.
65
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
66
+ #
67
+ def notify!(status)
68
+ tag = case status
69
+ when :success then '[Backup::Success]'
70
+ when :failure then '[Backup::Failure]'
71
+ when :warning then '[Backup::Warning]'
72
+ end
73
+ message = "#{ tag } #{ model.label } (#{ model.trigger })"
74
+
75
+ data = { :text => message }
76
+ [:channel, :username, :icon_emoji].each do |param|
77
+ val = send(param)
78
+ data.merge!(param => val) if val
79
+ end
80
+
81
+ data.merge!(:attachments => [attachment(status)])
82
+
83
+ options = {
84
+ :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
85
+ :body => URI.encode_www_form(:payload => JSON.dump(data))
86
+ }
87
+ options.merge!(:expects => 200) # raise error if unsuccessful
88
+ Excon.post(uri, options)
89
+ end
90
+
91
+ def attachment(status)
92
+ {
93
+ :fallback => "#{title(status)} - Job: #{model.label} (#{model.trigger})",
94
+ :text => title(status),
95
+ :color => color(status),
96
+ :fields => [
97
+ {
98
+ :title => "Job",
99
+ :value => "#{model.label} (#{model.trigger})",
100
+ :short => false
101
+ },
102
+ {
103
+ :title => "Started",
104
+ :value => model.started_at,
105
+ :short => true
106
+ },
107
+ {
108
+ :title => "Finished",
109
+ :value => model.finished_at,
110
+ :short => true
111
+ },
112
+ {
113
+ :title => "Duration",
114
+ :value => model.duration,
115
+ :short => true
116
+ },
117
+ {
118
+ :title => "Version",
119
+ :value => "Backup v#{Backup::VERSION}\nRuby: #{RUBY_DESCRIPTION}",
120
+ :short => false
121
+ },
122
+ log_field(status)
123
+ ].compact
124
+ }
125
+ end
126
+
127
+ def log_field(status)
128
+ send_log = send_log_on.include?(status)
129
+
130
+ return {
131
+ :title => "Detailed Backup Log",
132
+ :value => Logger.messages.map(&:formatted_lines).flatten.join("\n"),
133
+ :short => false,
134
+ } if send_log
135
+ end
136
+
137
+ def color(status)
138
+ case status
139
+ when :success then 'good'
140
+ when :failure then 'danger'
141
+ when :warning then 'warning'
142
+ end
143
+ end
144
+
145
+ def title(status)
146
+ case status
147
+ when :success then 'Backup Completed Successfully!'
148
+ when :failure then 'Backup Failed!'
149
+ when :warning then 'Backup Completed Successfully (with Warnings)!'
150
+ end
151
+ end
152
+
153
+ def uri
154
+ @uri ||= "https://#{team}.slack.com/services/hooks/incoming-webhook?token=#{token}"
155
+ end
156
+ end
157
+ end
158
+ end
@@ -0,0 +1,64 @@
1
+ # encoding: utf-8
2
+ require 'twitter'
3
+
4
+ module Backup
5
+ module Notifier
6
+ class Twitter < Base
7
+
8
+ ##
9
+ # Twitter consumer key credentials
10
+ attr_accessor :consumer_key, :consumer_secret
11
+
12
+ ##
13
+ # OAuth credentials
14
+ attr_accessor :oauth_token, :oauth_token_secret
15
+
16
+ def initialize(model, &block)
17
+ super
18
+ instance_eval(&block) if block_given?
19
+ end
20
+
21
+ private
22
+
23
+ ##
24
+ # Notify the user of the backup operation results.
25
+ #
26
+ # `status` indicates one of the following:
27
+ #
28
+ # `:success`
29
+ # : The backup completed successfully.
30
+ # : Notification will be sent if `on_success` is `true`.
31
+ #
32
+ # `:warning`
33
+ # : The backup completed successfully, but warnings were logged.
34
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
35
+ #
36
+ # `:failure`
37
+ # : The backup operation failed.
38
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
39
+ #
40
+ def notify!(status)
41
+ tag = case status
42
+ when :success then '[Backup::Success]'
43
+ when :warning then '[Backup::Warning]'
44
+ when :failure then '[Backup::Failure]'
45
+ end
46
+ message = "#{ tag } #{ model.label } (#{ model.trigger }) (@ #{ model.time })"
47
+ send_message(message)
48
+ end
49
+
50
+ # Twitter::Client will raise an error if unsuccessful.
51
+ def send_message(message)
52
+ client = ::Twitter::REST::Client.new do |config|
53
+ config.consumer_key = @consumer_key
54
+ config.consumer_secret = @consumer_secret
55
+ config.access_token = @oauth_token
56
+ config.access_token_secret = @oauth_token_secret
57
+ end
58
+
59
+ client.update(message)
60
+ end
61
+
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,68 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ module Notifier
5
+ class Zabbix < Base
6
+
7
+ attr_accessor :zabbix_host
8
+
9
+ attr_accessor :zabbix_port
10
+
11
+ attr_accessor :service_name
12
+
13
+ attr_accessor :service_host
14
+
15
+ attr_accessor :item_key
16
+
17
+ def initialize(model, &block)
18
+ super
19
+ instance_eval(&block) if block_given?
20
+
21
+ @zabbix_host ||= Config.hostname
22
+ @zabbix_port ||= 10051
23
+ @service_name ||= "Backup #{ model.trigger }"
24
+ @service_host ||= Config.hostname
25
+ @item_key ||= 'backup_status'
26
+ end
27
+
28
+ private
29
+
30
+ ##
31
+ # Notify the user of the backup operation results.
32
+ #
33
+ # `status` indicates one of the following:
34
+ #
35
+ # `:success`
36
+ # : The backup completed successfully.
37
+ # : Notification will be sent if `on_success` is `true`.
38
+ #
39
+ # `:warning`
40
+ # : The backup completed successfully, but warnings were logged.
41
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
42
+ #
43
+ # `:failure`
44
+ # : The backup operation failed.
45
+ # : Notification will be sent if `on_warning` or `on_success` is `true`.
46
+ #
47
+ def notify!(status)
48
+ message = case status
49
+ when :success then 'Completed Successfully'
50
+ when :warning then 'Completed Successfully (with Warnings)'
51
+ when :failure then 'Failed'
52
+ end
53
+ send_message("#{ message } in #{ model.duration }")
54
+ end
55
+
56
+ def send_message(message)
57
+ msg = [service_host, service_name, model.exit_status, message].join("\t")
58
+ cmd = "#{ utility(:zabbix_sender) }" +
59
+ " -z '#{ zabbix_host }'" +
60
+ " -p '#{ zabbix_port }'" +
61
+ " -s #{ service_host }" +
62
+ " -k #{ item_key }" +
63
+ " -o '#{ msg }'"
64
+ run("echo '#{ msg }' | #{ cmd }")
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,51 @@
1
+ # encoding: utf-8
2
+
3
+ module Backup
4
+ class Package
5
+
6
+ ##
7
+ # The time when the backup initiated (in format: 2011.02.20.03.29.59)
8
+ attr_accessor :time
9
+
10
+ ##
11
+ # The trigger which initiated the backup process
12
+ attr_reader :trigger
13
+
14
+ ##
15
+ # Extension for the final archive file(s)
16
+ attr_accessor :extension
17
+
18
+ ##
19
+ # Set by the Splitter if the final archive was "chunked"
20
+ attr_accessor :chunk_suffixes
21
+
22
+ ##
23
+ # If true, the Cycler will not attempt to remove the package when Cycling.
24
+ attr_accessor :no_cycle
25
+
26
+ ##
27
+ # The version of Backup used to create the package
28
+ attr_reader :version
29
+
30
+ def initialize(model)
31
+ @trigger = model.trigger
32
+ @extension = 'tar'
33
+ @chunk_suffixes = Array.new
34
+ @no_cycle = false
35
+ @version = VERSION
36
+ end
37
+
38
+ def filenames
39
+ if chunk_suffixes.empty?
40
+ [basename]
41
+ else
42
+ chunk_suffixes.map {|suffix| "#{ basename }-#{ suffix }" }
43
+ end
44
+ end
45
+
46
+ def basename
47
+ "#{ trigger }.#{ extension }"
48
+ end
49
+
50
+ end
51
+ end