ext_backup 5.0.0.beta.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (137) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +19 -0
  3. data/README.md +33 -0
  4. data/bin/backup +5 -0
  5. data/bin/docker_test +24 -0
  6. data/lib/backup.rb +140 -0
  7. data/lib/backup/archive.rb +169 -0
  8. data/lib/backup/binder.rb +18 -0
  9. data/lib/backup/cleaner.rb +112 -0
  10. data/lib/backup/cli.rb +370 -0
  11. data/lib/backup/cloud_io/base.rb +38 -0
  12. data/lib/backup/cloud_io/cloud_files.rb +296 -0
  13. data/lib/backup/cloud_io/s3.rb +253 -0
  14. data/lib/backup/compressor/base.rb +32 -0
  15. data/lib/backup/compressor/bzip2.rb +35 -0
  16. data/lib/backup/compressor/custom.rb +49 -0
  17. data/lib/backup/compressor/gzip.rb +73 -0
  18. data/lib/backup/config.rb +128 -0
  19. data/lib/backup/config/dsl.rb +102 -0
  20. data/lib/backup/config/helpers.rb +137 -0
  21. data/lib/backup/database/base.rb +86 -0
  22. data/lib/backup/database/mongodb.rb +186 -0
  23. data/lib/backup/database/mysql.rb +191 -0
  24. data/lib/backup/database/openldap.rb +93 -0
  25. data/lib/backup/database/postgresql.rb +132 -0
  26. data/lib/backup/database/redis.rb +176 -0
  27. data/lib/backup/database/riak.rb +79 -0
  28. data/lib/backup/database/sqlite.rb +55 -0
  29. data/lib/backup/encryptor/base.rb +27 -0
  30. data/lib/backup/encryptor/gpg.rb +737 -0
  31. data/lib/backup/encryptor/open_ssl.rb +74 -0
  32. data/lib/backup/errors.rb +53 -0
  33. data/lib/backup/logger.rb +197 -0
  34. data/lib/backup/logger/console.rb +48 -0
  35. data/lib/backup/logger/fog_adapter.rb +25 -0
  36. data/lib/backup/logger/logfile.rb +131 -0
  37. data/lib/backup/logger/syslog.rb +114 -0
  38. data/lib/backup/model.rb +472 -0
  39. data/lib/backup/notifier/base.rb +126 -0
  40. data/lib/backup/notifier/campfire.rb +61 -0
  41. data/lib/backup/notifier/command.rb +99 -0
  42. data/lib/backup/notifier/datadog.rb +104 -0
  43. data/lib/backup/notifier/flowdock.rb +99 -0
  44. data/lib/backup/notifier/hipchat.rb +116 -0
  45. data/lib/backup/notifier/http_post.rb +114 -0
  46. data/lib/backup/notifier/mail.rb +232 -0
  47. data/lib/backup/notifier/nagios.rb +65 -0
  48. data/lib/backup/notifier/pagerduty.rb +79 -0
  49. data/lib/backup/notifier/prowl.rb +68 -0
  50. data/lib/backup/notifier/pushover.rb +71 -0
  51. data/lib/backup/notifier/ses.rb +123 -0
  52. data/lib/backup/notifier/slack.rb +147 -0
  53. data/lib/backup/notifier/twitter.rb +55 -0
  54. data/lib/backup/notifier/zabbix.rb +60 -0
  55. data/lib/backup/package.rb +51 -0
  56. data/lib/backup/packager.rb +106 -0
  57. data/lib/backup/pipeline.rb +120 -0
  58. data/lib/backup/splitter.rb +73 -0
  59. data/lib/backup/storage/base.rb +66 -0
  60. data/lib/backup/storage/cloud_files.rb +156 -0
  61. data/lib/backup/storage/cycler.rb +70 -0
  62. data/lib/backup/storage/dropbox.rb +206 -0
  63. data/lib/backup/storage/ftp.rb +116 -0
  64. data/lib/backup/storage/local.rb +61 -0
  65. data/lib/backup/storage/qiniu.rb +65 -0
  66. data/lib/backup/storage/rsync.rb +246 -0
  67. data/lib/backup/storage/s3.rb +155 -0
  68. data/lib/backup/storage/scp.rb +65 -0
  69. data/lib/backup/storage/sftp.rb +80 -0
  70. data/lib/backup/syncer/base.rb +67 -0
  71. data/lib/backup/syncer/cloud/base.rb +176 -0
  72. data/lib/backup/syncer/cloud/cloud_files.rb +81 -0
  73. data/lib/backup/syncer/cloud/local_file.rb +97 -0
  74. data/lib/backup/syncer/cloud/s3.rb +109 -0
  75. data/lib/backup/syncer/rsync/base.rb +50 -0
  76. data/lib/backup/syncer/rsync/local.rb +27 -0
  77. data/lib/backup/syncer/rsync/pull.rb +47 -0
  78. data/lib/backup/syncer/rsync/push.rb +201 -0
  79. data/lib/backup/template.rb +41 -0
  80. data/lib/backup/utilities.rb +233 -0
  81. data/lib/backup/version.rb +3 -0
  82. data/lib/ext_backup.rb +5 -0
  83. data/lib/ext_backup/version.rb +5 -0
  84. data/templates/cli/archive +28 -0
  85. data/templates/cli/compressor/bzip2 +4 -0
  86. data/templates/cli/compressor/custom +7 -0
  87. data/templates/cli/compressor/gzip +4 -0
  88. data/templates/cli/config +123 -0
  89. data/templates/cli/databases/mongodb +15 -0
  90. data/templates/cli/databases/mysql +18 -0
  91. data/templates/cli/databases/openldap +24 -0
  92. data/templates/cli/databases/postgresql +16 -0
  93. data/templates/cli/databases/redis +16 -0
  94. data/templates/cli/databases/riak +17 -0
  95. data/templates/cli/databases/sqlite +11 -0
  96. data/templates/cli/encryptor/gpg +27 -0
  97. data/templates/cli/encryptor/openssl +9 -0
  98. data/templates/cli/model +26 -0
  99. data/templates/cli/notifier/zabbix +15 -0
  100. data/templates/cli/notifiers/campfire +12 -0
  101. data/templates/cli/notifiers/command +32 -0
  102. data/templates/cli/notifiers/datadog +57 -0
  103. data/templates/cli/notifiers/flowdock +16 -0
  104. data/templates/cli/notifiers/hipchat +16 -0
  105. data/templates/cli/notifiers/http_post +32 -0
  106. data/templates/cli/notifiers/mail +24 -0
  107. data/templates/cli/notifiers/nagios +13 -0
  108. data/templates/cli/notifiers/pagerduty +12 -0
  109. data/templates/cli/notifiers/prowl +11 -0
  110. data/templates/cli/notifiers/pushover +11 -0
  111. data/templates/cli/notifiers/ses +15 -0
  112. data/templates/cli/notifiers/slack +22 -0
  113. data/templates/cli/notifiers/twitter +13 -0
  114. data/templates/cli/splitter +7 -0
  115. data/templates/cli/storages/cloud_files +11 -0
  116. data/templates/cli/storages/dropbox +20 -0
  117. data/templates/cli/storages/ftp +13 -0
  118. data/templates/cli/storages/local +8 -0
  119. data/templates/cli/storages/qiniu +12 -0
  120. data/templates/cli/storages/rsync +17 -0
  121. data/templates/cli/storages/s3 +16 -0
  122. data/templates/cli/storages/scp +15 -0
  123. data/templates/cli/storages/sftp +15 -0
  124. data/templates/cli/syncers/cloud_files +22 -0
  125. data/templates/cli/syncers/rsync_local +20 -0
  126. data/templates/cli/syncers/rsync_pull +28 -0
  127. data/templates/cli/syncers/rsync_push +28 -0
  128. data/templates/cli/syncers/s3 +27 -0
  129. data/templates/general/links +3 -0
  130. data/templates/general/version.erb +2 -0
  131. data/templates/notifier/mail/failure.erb +16 -0
  132. data/templates/notifier/mail/success.erb +16 -0
  133. data/templates/notifier/mail/warning.erb +16 -0
  134. data/templates/storage/dropbox/authorization_url.erb +6 -0
  135. data/templates/storage/dropbox/authorized.erb +4 -0
  136. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  137. metadata +506 -0
@@ -0,0 +1,41 @@
1
+ require "erb"
2
+
3
+ module Backup
4
+ class Template
5
+ # Holds a binding object. Nil if not provided.
6
+ attr_accessor :binding
7
+
8
+ ##
9
+ # Creates a new instance of the Backup::Template class
10
+ # and optionally takes an argument that can be either a binding object, a Hash or nil
11
+ def initialize(object = nil)
12
+ @binding =
13
+ if object.is_a?(Binding)
14
+ object
15
+ elsif object.is_a?(Hash)
16
+ Backup::Binder.new(object).get_binding
17
+ end
18
+ end
19
+
20
+ ##
21
+ # Renders the provided file (in the context of the binding if any) to the console
22
+ def render(file)
23
+ puts result(file)
24
+ end
25
+
26
+ ##
27
+ # Returns a String object containing the contents of the file (in the context of the binding if any)
28
+ def result(file)
29
+ ERB.new(file_contents(file), nil, "<>").result(binding)
30
+ end
31
+
32
+ private
33
+
34
+ ##
35
+ # Reads and returns the contents of the provided file path,
36
+ # relative from the Backup::TEMPLATE_PATH
37
+ def file_contents(file)
38
+ File.read(File.join(Backup::TEMPLATE_PATH, file))
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,233 @@
1
+ module Backup
2
+ module Utilities
3
+ class Error < Backup::Error; end
4
+
5
+ UTILITIES_NAMES = %w[
6
+ tar cat split sudo chown hostname
7
+ gzip bzip2
8
+ mongo mongodump mysqldump innobackupex
9
+ pg_dump pg_dumpall redis-cli riak-admin
10
+ gpg openssl
11
+ rsync ssh
12
+ sendmail exim
13
+ send_nsca
14
+ zabbix_sender
15
+ ].freeze
16
+
17
+ # @api private
18
+ class DSL
19
+ def initialize(utils)
20
+ @utilities = utils
21
+ end
22
+
23
+ # Helper methods to allow users to set the path for all utilities in the
24
+ # .configure block.
25
+ #
26
+ # Utility names with dashes (`redis-cli`) will be set using method calls
27
+ # with an underscore (`redis_cli`).
28
+ UTILITIES_NAMES.each do |util_name|
29
+ define_method util_name.tr("-", "_") do |raw_path|
30
+ path = File.expand_path(raw_path)
31
+
32
+ unless File.executable?(path)
33
+ raise Utilities::Error, <<-EOS
34
+ The path given for '#{util_name}' was not found or not executable.
35
+ Path was: #{path}
36
+ EOS
37
+ end
38
+
39
+ @utilities.utilities[util_name] = path
40
+ end
41
+ end
42
+
43
+ # Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
44
+ def tar_dist(val)
45
+ Utilities.tar_dist(val)
46
+ end
47
+ end
48
+
49
+ class << self
50
+ ##
51
+ # Configure the path to system utilities used by Backup.
52
+ #
53
+ # Backup will attempt to locate any required system utilities using a
54
+ # +which+ command call. If a utility can not be found, or you need to
55
+ # specify an alternate path for a utility, you may do so in your
56
+ # +config.rb+ file using this method.
57
+ #
58
+ # Backup supports both GNU and BSD utilities.
59
+ # While Backup uses these utilities in a manner compatible with either
60
+ # version, the +tar+ utility requires some special handling with respect
61
+ # to +Archive+s. Backup will attempt to detect if the +tar+ command
62
+ # found (or set here) is GNU or BSD. If for some reason this fails,
63
+ # this may be set using the +tar_dist+ command shown below.
64
+ #
65
+ # Backup::Utilities.configure do
66
+ # # General Utilites
67
+ # tar '/path/to/tar'
68
+ # tar_dist :gnu # or :bsd
69
+ # cat '/path/to/cat'
70
+ # split '/path/to/split'
71
+ # sudo '/path/to/sudo'
72
+ # chown '/path/to/chown'
73
+ # hostname '/path/to/hostname'
74
+ #
75
+ # # Compressors
76
+ # gzip '/path/to/gzip'
77
+ # bzip2 '/path/to/bzip2'
78
+ #
79
+ # # Database Utilities
80
+ # mongo '/path/to/mongo'
81
+ # mongodump '/path/to/mongodump'
82
+ # mysqldump '/path/to/mysqldump'
83
+ # pg_dump '/path/to/pg_dump'
84
+ # pg_dumpall '/path/to/pg_dumpall'
85
+ # redis_cli '/path/to/redis-cli'
86
+ # riak_admin '/path/to/riak-admin'
87
+ #
88
+ # # Encryptors
89
+ # gpg '/path/to/gpg'
90
+ # openssl '/path/to/openssl'
91
+ #
92
+ # # Syncer and Storage
93
+ # rsync '/path/to/rsync'
94
+ # ssh '/path/to/ssh'
95
+ #
96
+ # # Notifiers
97
+ # sendmail '/path/to/sendmail'
98
+ # exim '/path/to/exim'
99
+ # send_nsca '/path/to/send_nsca'
100
+ # zabbix_sender '/path/to/zabbix_sender'
101
+ # end
102
+ #
103
+ # These paths may be set using absolute paths, or relative to the
104
+ # working directory when Backup is run.
105
+ def configure(&block)
106
+ DSL.new(self).instance_eval(&block)
107
+ end
108
+
109
+ def tar_dist(val)
110
+ # the acceptance tests need to be able to reset this to nil
111
+ @gnu_tar = val.nil? ? nil : val == :gnu
112
+ end
113
+
114
+ def gnu_tar?
115
+ return @gnu_tar unless @gnu_tar.nil?
116
+ @gnu_tar = !!run("#{utility(:tar)} --version").match(/GNU/)
117
+ end
118
+
119
+ def utilities
120
+ @utilities ||= {}
121
+ end
122
+
123
+ private
124
+
125
+ ##
126
+ # Returns the full path to the specified utility.
127
+ # Raises an error if utility can not be found in the system's $PATH
128
+ def utility(name)
129
+ name = name.to_s.strip
130
+ raise Error, "Utility Name Empty" if name.empty?
131
+
132
+ utilities[name] ||= `which '#{name}' 2>/dev/null`.chomp
133
+ raise Error, <<-EOS if utilities[name].empty?
134
+ Could not locate '#{name}'.
135
+ Make sure the specified utility is installed
136
+ and available in your system's $PATH, or specify it's location
137
+ in your 'config.rb' file using Backup::Utilities.configure
138
+ EOS
139
+
140
+ utilities[name].dup
141
+ end
142
+
143
+ ##
144
+ # Returns the name of the command name from the given command line.
145
+ # This is only used to simplify log messages.
146
+ def command_name(command)
147
+ parts = []
148
+ command = command.split(" ")
149
+ command.shift while command[0].to_s.include?("=")
150
+ parts << command.shift.split("/")[-1]
151
+ if parts[0] == "sudo"
152
+ until command.empty?
153
+ part = command.shift
154
+ if part.include?("/")
155
+ parts << part.split("/")[-1]
156
+ break
157
+ else
158
+ parts << part
159
+ end
160
+ end
161
+ end
162
+ parts.join(" ")
163
+ end
164
+
165
+ ##
166
+ # Runs a system command
167
+ #
168
+ # All messages generated by the command will be logged.
169
+ # Messages on STDERR will be logged as warnings.
170
+ #
171
+ # If the command fails to execute, or returns a non-zero exit status
172
+ # an Error will be raised.
173
+ #
174
+ # Returns STDOUT
175
+ def run(command)
176
+ name = command_name(command)
177
+ Logger.info "Running system utility '#{name}'..."
178
+
179
+ begin
180
+ out = ""
181
+ err = ""
182
+ ps = Open4.popen4(command) do |_pid, stdin, stdout, stderr|
183
+ stdin.close
184
+ out = stdout.read.strip
185
+ err = stderr.read.strip
186
+ end
187
+ rescue Exception => e
188
+ raise Error.wrap(e, "Failed to execute '#{name}'")
189
+ end
190
+
191
+ unless ps.success?
192
+ raise Error, <<-EOS
193
+ '#{name}' failed with exit status: #{ps.exitstatus}
194
+ STDOUT Messages: #{out.empty? ? "None" : "\n#{out}"}
195
+ STDERR Messages: #{err.empty? ? "None" : "\n#{err}"}
196
+ EOS
197
+ end
198
+
199
+ unless out.empty?
200
+ Logger.info(out.lines.map { |line| "#{name}:STDOUT: #{line}" }.join)
201
+ end
202
+
203
+ unless err.empty?
204
+ Logger.warn(err.lines.map { |line| "#{name}:STDERR: #{line}" }.join)
205
+ end
206
+
207
+ out
208
+ end
209
+
210
+ def reset!
211
+ utilities.clear
212
+ @gnu_tar = nil
213
+ end
214
+ end
215
+
216
+ # Allows these utility methods to be included in other classes,
217
+ # while allowing them to be stubbed in spec_helper for all specs.
218
+ module Helpers
219
+ [:utility, :command_name, :run].each do |name|
220
+ define_method name do |arg|
221
+ Utilities.send(name, arg)
222
+ end
223
+ private name
224
+ end
225
+
226
+ private
227
+
228
+ def gnu_tar?
229
+ Utilities.gnu_tar?
230
+ end
231
+ end
232
+ end
233
+ end
@@ -0,0 +1,3 @@
1
+ module Backup
2
+ VERSION = "5.0.0.beta.2"
3
+ end
@@ -0,0 +1,5 @@
1
+ require 'backup'
2
+
3
+ module ExtBackup
4
+ # Your code goes here...
5
+ end
@@ -0,0 +1,5 @@
1
+ require 'backup/version'
2
+
3
+ module ExtBackup
4
+ VERSION = "#{Backup::VERSION}.1"
5
+ end
@@ -0,0 +1,28 @@
1
+ ##
2
+ # Archive [Archive]
3
+ #
4
+ # Adding a file or directory (including sub-directories):
5
+ # archive.add "/path/to/a/file.rb"
6
+ # archive.add "/path/to/a/directory/"
7
+ #
8
+ # Excluding a file or directory (including sub-directories):
9
+ # archive.exclude "/path/to/an/excluded_file.rb"
10
+ # archive.exclude "/path/to/an/excluded_directory
11
+ #
12
+ # By default, relative paths will be relative to the directory
13
+ # where `backup perform` is executed, and they will be expanded
14
+ # to the root of the filesystem when added to the archive.
15
+ #
16
+ # If a `root` path is set, relative paths will be relative to the
17
+ # given `root` path and will not be expanded when added to the archive.
18
+ #
19
+ # archive.root '/path/to/archive/root'
20
+ #
21
+ archive :my_archive do |archive|
22
+ # Run the `tar` command using `sudo`
23
+ # archive.use_sudo
24
+ archive.add "/path/to/a/file.rb"
25
+ archive.add "/path/to/a/folder/"
26
+ archive.exclude "/path/to/a/excluded_file.rb"
27
+ archive.exclude "/path/to/a/excluded_folder"
28
+ end
@@ -0,0 +1,4 @@
1
+ ##
2
+ # Bzip2 [Compressor]
3
+ #
4
+ compress_with Bzip2
@@ -0,0 +1,7 @@
1
+ ##
2
+ # Custom [Compressor]
3
+ #
4
+ compress_with Custom do |compressor|
5
+ compressor.command = 'gzip'
6
+ compressor.extension = '.gz'
7
+ end
@@ -0,0 +1,4 @@
1
+ ##
2
+ # Gzip [Compressor]
3
+ #
4
+ compress_with Gzip
@@ -0,0 +1,123 @@
1
+ # encoding: utf-8
2
+
3
+ ##
4
+ # Backup v<%= Backup::VERSION.split('.').first %>.x Configuration
5
+ #
6
+ # Documentation: http://backup.github.io/backup
7
+ # Issue Tracker: https://github.com/backup/backup/issues
8
+
9
+ ##
10
+ # Config Options
11
+ #
12
+ # The options here may be overridden on the command line, but the result
13
+ # will depend on the use of --root-path on the command line.
14
+ #
15
+ # If --root-path is used on the command line, then all paths set here
16
+ # will be overridden. If a path (like --tmp-path) is not given along with
17
+ # --root-path, that path will use it's default location _relative to --root-path_.
18
+ #
19
+ # If --root-path is not used on the command line, a path option (like --tmp-path)
20
+ # given on the command line will override the tmp_path set here, but all other
21
+ # paths set here will be used.
22
+ #
23
+ # Note that relative paths given on the command line without --root-path
24
+ # are relative to the current directory. The root_path set here only applies
25
+ # to relative paths set here.
26
+ #
27
+ # ---
28
+ #
29
+ # Sets the root path for all relative paths, including default paths.
30
+ # May be an absolute path, or relative to the current working directory.
31
+ #
32
+ # root_path 'my/root'
33
+ #
34
+ # Sets the path where backups are processed until they're stored.
35
+ # This must have enough free space to hold apx. 2 backups.
36
+ # May be an absolute path, or relative to the current directory or +root_path+.
37
+ #
38
+ # tmp_path 'my/tmp'
39
+ #
40
+ # Sets the path where backup stores persistent information.
41
+ # When Backup's Cycler is used, small YAML files are stored here.
42
+ # May be an absolute path, or relative to the current directory or +root_path+.
43
+ #
44
+ # data_path 'my/data'
45
+
46
+ ##
47
+ # Utilities
48
+ #
49
+ # If you need to use a utility other than the one Backup detects,
50
+ # or a utility can not be found in your $PATH.
51
+ #
52
+ # Utilities.configure do
53
+ # tar '/usr/bin/gnutar'
54
+ # redis_cli '/opt/redis/redis-cli'
55
+ # end
56
+
57
+ ##
58
+ # Logging
59
+ #
60
+ # Logging options may be set on the command line, but certain settings
61
+ # may only be configured here.
62
+ #
63
+ # Logger.configure do
64
+ # console.quiet = true # Same as command line: --quiet
65
+ # logfile.max_bytes = 2_000_000 # Default: 500_000
66
+ # syslog.enabled = true # Same as command line: --syslog
67
+ # syslog.ident = 'my_app_backup' # Default: 'backup'
68
+ # end
69
+ #
70
+ # Command line options will override those set here.
71
+ # For example, the following would override the example settings above
72
+ # to disable syslog and enable console output.
73
+ # backup perform --trigger my_backup --no-syslog --no-quiet
74
+
75
+ ##
76
+ # Component Defaults
77
+ #
78
+ # Set default options to be applied to components in all models.
79
+ # Options set within a model will override those set here.
80
+ #
81
+ # Storage::S3.defaults do |s3|
82
+ # s3.access_key_id = "my_access_key_id"
83
+ # s3.secret_access_key = "my_secret_access_key"
84
+ # end
85
+ #
86
+ # Notifier::Mail.defaults do |mail|
87
+ # mail.from = 'sender@email.com'
88
+ # mail.to = 'receiver@email.com'
89
+ # mail.address = 'smtp.gmail.com'
90
+ # mail.port = 587
91
+ # mail.domain = 'your.host.name'
92
+ # mail.user_name = 'sender@email.com'
93
+ # mail.password = 'my_password'
94
+ # mail.authentication = 'plain'
95
+ # mail.encryption = :starttls
96
+ # end
97
+
98
+ ##
99
+ # Preconfigured Models
100
+ #
101
+ # Create custom models with preconfigured components.
102
+ # Components added within the model definition will
103
+ # +add to+ the preconfigured components.
104
+ #
105
+ # preconfigure 'MyModel' do
106
+ # archive :user_pictures do |archive|
107
+ # archive.add '~/pictures'
108
+ # end
109
+ #
110
+ # notify_by Mail do |mail|
111
+ # mail.to = 'admin@email.com'
112
+ # end
113
+ # end
114
+ #
115
+ # MyModel.new(:john_smith, 'John Smith Backup') do
116
+ # archive :user_music do |archive|
117
+ # archive.add '~/music'
118
+ # end
119
+ #
120
+ # notify_by Mail do |mail|
121
+ # mail.to = 'john.smith@email.com'
122
+ # end
123
+ # end