ext_backup 5.0.0.beta.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +19 -0
  3. data/README.md +33 -0
  4. data/bin/backup +5 -0
  5. data/bin/docker_test +24 -0
  6. data/lib/backup.rb +140 -0
  7. data/lib/backup/archive.rb +169 -0
  8. data/lib/backup/binder.rb +18 -0
  9. data/lib/backup/cleaner.rb +112 -0
  10. data/lib/backup/cli.rb +370 -0
  11. data/lib/backup/cloud_io/base.rb +38 -0
  12. data/lib/backup/cloud_io/cloud_files.rb +296 -0
  13. data/lib/backup/cloud_io/s3.rb +253 -0
  14. data/lib/backup/compressor/base.rb +32 -0
  15. data/lib/backup/compressor/bzip2.rb +35 -0
  16. data/lib/backup/compressor/custom.rb +49 -0
  17. data/lib/backup/compressor/gzip.rb +73 -0
  18. data/lib/backup/config.rb +128 -0
  19. data/lib/backup/config/dsl.rb +102 -0
  20. data/lib/backup/config/helpers.rb +137 -0
  21. data/lib/backup/database/base.rb +86 -0
  22. data/lib/backup/database/mongodb.rb +186 -0
  23. data/lib/backup/database/mysql.rb +191 -0
  24. data/lib/backup/database/openldap.rb +93 -0
  25. data/lib/backup/database/postgresql.rb +132 -0
  26. data/lib/backup/database/redis.rb +176 -0
  27. data/lib/backup/database/riak.rb +79 -0
  28. data/lib/backup/database/sqlite.rb +55 -0
  29. data/lib/backup/encryptor/base.rb +27 -0
  30. data/lib/backup/encryptor/gpg.rb +737 -0
  31. data/lib/backup/encryptor/open_ssl.rb +74 -0
  32. data/lib/backup/errors.rb +53 -0
  33. data/lib/backup/logger.rb +197 -0
  34. data/lib/backup/logger/console.rb +48 -0
  35. data/lib/backup/logger/fog_adapter.rb +25 -0
  36. data/lib/backup/logger/logfile.rb +131 -0
  37. data/lib/backup/logger/syslog.rb +114 -0
  38. data/lib/backup/model.rb +472 -0
  39. data/lib/backup/notifier/base.rb +126 -0
  40. data/lib/backup/notifier/campfire.rb +61 -0
  41. data/lib/backup/notifier/command.rb +99 -0
  42. data/lib/backup/notifier/datadog.rb +104 -0
  43. data/lib/backup/notifier/flowdock.rb +99 -0
  44. data/lib/backup/notifier/hipchat.rb +116 -0
  45. data/lib/backup/notifier/http_post.rb +114 -0
  46. data/lib/backup/notifier/mail.rb +232 -0
  47. data/lib/backup/notifier/nagios.rb +65 -0
  48. data/lib/backup/notifier/pagerduty.rb +79 -0
  49. data/lib/backup/notifier/prowl.rb +68 -0
  50. data/lib/backup/notifier/pushover.rb +71 -0
  51. data/lib/backup/notifier/ses.rb +123 -0
  52. data/lib/backup/notifier/slack.rb +147 -0
  53. data/lib/backup/notifier/twitter.rb +55 -0
  54. data/lib/backup/notifier/zabbix.rb +60 -0
  55. data/lib/backup/package.rb +51 -0
  56. data/lib/backup/packager.rb +106 -0
  57. data/lib/backup/pipeline.rb +120 -0
  58. data/lib/backup/splitter.rb +73 -0
  59. data/lib/backup/storage/base.rb +66 -0
  60. data/lib/backup/storage/cloud_files.rb +156 -0
  61. data/lib/backup/storage/cycler.rb +70 -0
  62. data/lib/backup/storage/dropbox.rb +206 -0
  63. data/lib/backup/storage/ftp.rb +116 -0
  64. data/lib/backup/storage/local.rb +61 -0
  65. data/lib/backup/storage/qiniu.rb +65 -0
  66. data/lib/backup/storage/rsync.rb +246 -0
  67. data/lib/backup/storage/s3.rb +155 -0
  68. data/lib/backup/storage/scp.rb +65 -0
  69. data/lib/backup/storage/sftp.rb +80 -0
  70. data/lib/backup/syncer/base.rb +67 -0
  71. data/lib/backup/syncer/cloud/base.rb +176 -0
  72. data/lib/backup/syncer/cloud/cloud_files.rb +81 -0
  73. data/lib/backup/syncer/cloud/local_file.rb +97 -0
  74. data/lib/backup/syncer/cloud/s3.rb +109 -0
  75. data/lib/backup/syncer/rsync/base.rb +50 -0
  76. data/lib/backup/syncer/rsync/local.rb +27 -0
  77. data/lib/backup/syncer/rsync/pull.rb +47 -0
  78. data/lib/backup/syncer/rsync/push.rb +201 -0
  79. data/lib/backup/template.rb +41 -0
  80. data/lib/backup/utilities.rb +233 -0
  81. data/lib/backup/version.rb +3 -0
  82. data/lib/ext_backup.rb +5 -0
  83. data/lib/ext_backup/version.rb +5 -0
  84. data/templates/cli/archive +28 -0
  85. data/templates/cli/compressor/bzip2 +4 -0
  86. data/templates/cli/compressor/custom +7 -0
  87. data/templates/cli/compressor/gzip +4 -0
  88. data/templates/cli/config +123 -0
  89. data/templates/cli/databases/mongodb +15 -0
  90. data/templates/cli/databases/mysql +18 -0
  91. data/templates/cli/databases/openldap +24 -0
  92. data/templates/cli/databases/postgresql +16 -0
  93. data/templates/cli/databases/redis +16 -0
  94. data/templates/cli/databases/riak +17 -0
  95. data/templates/cli/databases/sqlite +11 -0
  96. data/templates/cli/encryptor/gpg +27 -0
  97. data/templates/cli/encryptor/openssl +9 -0
  98. data/templates/cli/model +26 -0
  99. data/templates/cli/notifier/zabbix +15 -0
  100. data/templates/cli/notifiers/campfire +12 -0
  101. data/templates/cli/notifiers/command +32 -0
  102. data/templates/cli/notifiers/datadog +57 -0
  103. data/templates/cli/notifiers/flowdock +16 -0
  104. data/templates/cli/notifiers/hipchat +16 -0
  105. data/templates/cli/notifiers/http_post +32 -0
  106. data/templates/cli/notifiers/mail +24 -0
  107. data/templates/cli/notifiers/nagios +13 -0
  108. data/templates/cli/notifiers/pagerduty +12 -0
  109. data/templates/cli/notifiers/prowl +11 -0
  110. data/templates/cli/notifiers/pushover +11 -0
  111. data/templates/cli/notifiers/ses +15 -0
  112. data/templates/cli/notifiers/slack +22 -0
  113. data/templates/cli/notifiers/twitter +13 -0
  114. data/templates/cli/splitter +7 -0
  115. data/templates/cli/storages/cloud_files +11 -0
  116. data/templates/cli/storages/dropbox +20 -0
  117. data/templates/cli/storages/ftp +13 -0
  118. data/templates/cli/storages/local +8 -0
  119. data/templates/cli/storages/qiniu +12 -0
  120. data/templates/cli/storages/rsync +17 -0
  121. data/templates/cli/storages/s3 +16 -0
  122. data/templates/cli/storages/scp +15 -0
  123. data/templates/cli/storages/sftp +15 -0
  124. data/templates/cli/syncers/cloud_files +22 -0
  125. data/templates/cli/syncers/rsync_local +20 -0
  126. data/templates/cli/syncers/rsync_pull +28 -0
  127. data/templates/cli/syncers/rsync_push +28 -0
  128. data/templates/cli/syncers/s3 +27 -0
  129. data/templates/general/links +3 -0
  130. data/templates/general/version.erb +2 -0
  131. data/templates/notifier/mail/failure.erb +16 -0
  132. data/templates/notifier/mail/success.erb +16 -0
  133. data/templates/notifier/mail/warning.erb +16 -0
  134. data/templates/storage/dropbox/authorization_url.erb +6 -0
  135. data/templates/storage/dropbox/authorized.erb +4 -0
  136. data/templates/storage/dropbox/cache_file_written.erb +10 -0
  137. metadata +506 -0
@@ -0,0 +1,41 @@
1
+ require "erb"
2
+
3
+ module Backup
4
+ class Template
5
+ # Holds a binding object. Nil if not provided.
6
+ attr_accessor :binding
7
+
8
+ ##
9
+ # Creates a new instance of the Backup::Template class
10
+ # and optionally takes an argument that can be either a binding object, a Hash or nil
11
+ def initialize(object = nil)
12
+ @binding =
13
+ if object.is_a?(Binding)
14
+ object
15
+ elsif object.is_a?(Hash)
16
+ Backup::Binder.new(object).get_binding
17
+ end
18
+ end
19
+
20
+ ##
21
+ # Renders the provided file (in the context of the binding if any) to the console
22
+ def render(file)
23
+ puts result(file)
24
+ end
25
+
26
+ ##
27
+ # Returns a String object containing the contents of the file (in the context of the binding if any)
28
+ def result(file)
29
+ ERB.new(file_contents(file), nil, "<>").result(binding)
30
+ end
31
+
32
+ private
33
+
34
+ ##
35
+ # Reads and returns the contents of the provided file path,
36
+ # relative from the Backup::TEMPLATE_PATH
37
+ def file_contents(file)
38
+ File.read(File.join(Backup::TEMPLATE_PATH, file))
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,233 @@
1
+ module Backup
2
+ module Utilities
3
+ class Error < Backup::Error; end
4
+
5
+ UTILITIES_NAMES = %w[
6
+ tar cat split sudo chown hostname
7
+ gzip bzip2
8
+ mongo mongodump mysqldump innobackupex
9
+ pg_dump pg_dumpall redis-cli riak-admin
10
+ gpg openssl
11
+ rsync ssh
12
+ sendmail exim
13
+ send_nsca
14
+ zabbix_sender
15
+ ].freeze
16
+
17
+ # @api private
18
+ class DSL
19
+ def initialize(utils)
20
+ @utilities = utils
21
+ end
22
+
23
+ # Helper methods to allow users to set the path for all utilities in the
24
+ # .configure block.
25
+ #
26
+ # Utility names with dashes (`redis-cli`) will be set using method calls
27
+ # with an underscore (`redis_cli`).
28
+ UTILITIES_NAMES.each do |util_name|
29
+ define_method util_name.tr("-", "_") do |raw_path|
30
+ path = File.expand_path(raw_path)
31
+
32
+ unless File.executable?(path)
33
+ raise Utilities::Error, <<-EOS
34
+ The path given for '#{util_name}' was not found or not executable.
35
+ Path was: #{path}
36
+ EOS
37
+ end
38
+
39
+ @utilities.utilities[util_name] = path
40
+ end
41
+ end
42
+
43
+ # Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
44
+ def tar_dist(val)
45
+ Utilities.tar_dist(val)
46
+ end
47
+ end
48
+
49
+ class << self
50
+ ##
51
+ # Configure the path to system utilities used by Backup.
52
+ #
53
+ # Backup will attempt to locate any required system utilities using a
54
+ # +which+ command call. If a utility can not be found, or you need to
55
+ # specify an alternate path for a utility, you may do so in your
56
+ # +config.rb+ file using this method.
57
+ #
58
+ # Backup supports both GNU and BSD utilities.
59
+ # While Backup uses these utilities in a manner compatible with either
60
+ # version, the +tar+ utility requires some special handling with respect
61
+ # to +Archive+s. Backup will attempt to detect if the +tar+ command
62
+ # found (or set here) is GNU or BSD. If for some reason this fails,
63
+ # this may be set using the +tar_dist+ command shown below.
64
+ #
65
+ # Backup::Utilities.configure do
66
+ # # General Utilites
67
+ # tar '/path/to/tar'
68
+ # tar_dist :gnu # or :bsd
69
+ # cat '/path/to/cat'
70
+ # split '/path/to/split'
71
+ # sudo '/path/to/sudo'
72
+ # chown '/path/to/chown'
73
+ # hostname '/path/to/hostname'
74
+ #
75
+ # # Compressors
76
+ # gzip '/path/to/gzip'
77
+ # bzip2 '/path/to/bzip2'
78
+ #
79
+ # # Database Utilities
80
+ # mongo '/path/to/mongo'
81
+ # mongodump '/path/to/mongodump'
82
+ # mysqldump '/path/to/mysqldump'
83
+ # pg_dump '/path/to/pg_dump'
84
+ # pg_dumpall '/path/to/pg_dumpall'
85
+ # redis_cli '/path/to/redis-cli'
86
+ # riak_admin '/path/to/riak-admin'
87
+ #
88
+ # # Encryptors
89
+ # gpg '/path/to/gpg'
90
+ # openssl '/path/to/openssl'
91
+ #
92
+ # # Syncer and Storage
93
+ # rsync '/path/to/rsync'
94
+ # ssh '/path/to/ssh'
95
+ #
96
+ # # Notifiers
97
+ # sendmail '/path/to/sendmail'
98
+ # exim '/path/to/exim'
99
+ # send_nsca '/path/to/send_nsca'
100
+ # zabbix_sender '/path/to/zabbix_sender'
101
+ # end
102
+ #
103
+ # These paths may be set using absolute paths, or relative to the
104
+ # working directory when Backup is run.
105
+ def configure(&block)
106
+ DSL.new(self).instance_eval(&block)
107
+ end
108
+
109
+ def tar_dist(val)
110
+ # the acceptance tests need to be able to reset this to nil
111
+ @gnu_tar = val.nil? ? nil : val == :gnu
112
+ end
113
+
114
+ def gnu_tar?
115
+ return @gnu_tar unless @gnu_tar.nil?
116
+ @gnu_tar = !!run("#{utility(:tar)} --version").match(/GNU/)
117
+ end
118
+
119
+ def utilities
120
+ @utilities ||= {}
121
+ end
122
+
123
+ private
124
+
125
+ ##
126
+ # Returns the full path to the specified utility.
127
+ # Raises an error if utility can not be found in the system's $PATH
128
+ def utility(name)
129
+ name = name.to_s.strip
130
+ raise Error, "Utility Name Empty" if name.empty?
131
+
132
+ utilities[name] ||= `which '#{name}' 2>/dev/null`.chomp
133
+ raise Error, <<-EOS if utilities[name].empty?
134
+ Could not locate '#{name}'.
135
+ Make sure the specified utility is installed
136
+ and available in your system's $PATH, or specify it's location
137
+ in your 'config.rb' file using Backup::Utilities.configure
138
+ EOS
139
+
140
+ utilities[name].dup
141
+ end
142
+
143
+ ##
144
+ # Returns the name of the command name from the given command line.
145
+ # This is only used to simplify log messages.
146
+ def command_name(command)
147
+ parts = []
148
+ command = command.split(" ")
149
+ command.shift while command[0].to_s.include?("=")
150
+ parts << command.shift.split("/")[-1]
151
+ if parts[0] == "sudo"
152
+ until command.empty?
153
+ part = command.shift
154
+ if part.include?("/")
155
+ parts << part.split("/")[-1]
156
+ break
157
+ else
158
+ parts << part
159
+ end
160
+ end
161
+ end
162
+ parts.join(" ")
163
+ end
164
+
165
+ ##
166
+ # Runs a system command
167
+ #
168
+ # All messages generated by the command will be logged.
169
+ # Messages on STDERR will be logged as warnings.
170
+ #
171
+ # If the command fails to execute, or returns a non-zero exit status
172
+ # an Error will be raised.
173
+ #
174
+ # Returns STDOUT
175
+ def run(command)
176
+ name = command_name(command)
177
+ Logger.info "Running system utility '#{name}'..."
178
+
179
+ begin
180
+ out = ""
181
+ err = ""
182
+ ps = Open4.popen4(command) do |_pid, stdin, stdout, stderr|
183
+ stdin.close
184
+ out = stdout.read.strip
185
+ err = stderr.read.strip
186
+ end
187
+ rescue Exception => e
188
+ raise Error.wrap(e, "Failed to execute '#{name}'")
189
+ end
190
+
191
+ unless ps.success?
192
+ raise Error, <<-EOS
193
+ '#{name}' failed with exit status: #{ps.exitstatus}
194
+ STDOUT Messages: #{out.empty? ? "None" : "\n#{out}"}
195
+ STDERR Messages: #{err.empty? ? "None" : "\n#{err}"}
196
+ EOS
197
+ end
198
+
199
+ unless out.empty?
200
+ Logger.info(out.lines.map { |line| "#{name}:STDOUT: #{line}" }.join)
201
+ end
202
+
203
+ unless err.empty?
204
+ Logger.warn(err.lines.map { |line| "#{name}:STDERR: #{line}" }.join)
205
+ end
206
+
207
+ out
208
+ end
209
+
210
+ def reset!
211
+ utilities.clear
212
+ @gnu_tar = nil
213
+ end
214
+ end
215
+
216
+ # Allows these utility methods to be included in other classes,
217
+ # while allowing them to be stubbed in spec_helper for all specs.
218
+ module Helpers
219
+ [:utility, :command_name, :run].each do |name|
220
+ define_method name do |arg|
221
+ Utilities.send(name, arg)
222
+ end
223
+ private name
224
+ end
225
+
226
+ private
227
+
228
+ def gnu_tar?
229
+ Utilities.gnu_tar?
230
+ end
231
+ end
232
+ end
233
+ end
@@ -0,0 +1,3 @@
1
+ module Backup
2
+ VERSION = "5.0.0.beta.2"
3
+ end
@@ -0,0 +1,5 @@
1
+ require 'backup'
2
+
3
+ module ExtBackup
4
+ # Your code goes here...
5
+ end
@@ -0,0 +1,5 @@
1
+ require 'backup/version'
2
+
3
+ module ExtBackup
4
+ VERSION = "#{Backup::VERSION}.1"
5
+ end
@@ -0,0 +1,28 @@
1
+ ##
2
+ # Archive [Archive]
3
+ #
4
+ # Adding a file or directory (including sub-directories):
5
+ # archive.add "/path/to/a/file.rb"
6
+ # archive.add "/path/to/a/directory/"
7
+ #
8
+ # Excluding a file or directory (including sub-directories):
9
+ # archive.exclude "/path/to/an/excluded_file.rb"
10
+ # archive.exclude "/path/to/an/excluded_directory
11
+ #
12
+ # By default, relative paths will be relative to the directory
13
+ # where `backup perform` is executed, and they will be expanded
14
+ # to the root of the filesystem when added to the archive.
15
+ #
16
+ # If a `root` path is set, relative paths will be relative to the
17
+ # given `root` path and will not be expanded when added to the archive.
18
+ #
19
+ # archive.root '/path/to/archive/root'
20
+ #
21
+ archive :my_archive do |archive|
22
+ # Run the `tar` command using `sudo`
23
+ # archive.use_sudo
24
+ archive.add "/path/to/a/file.rb"
25
+ archive.add "/path/to/a/folder/"
26
+ archive.exclude "/path/to/a/excluded_file.rb"
27
+ archive.exclude "/path/to/a/excluded_folder"
28
+ end
@@ -0,0 +1,4 @@
1
+ ##
2
+ # Bzip2 [Compressor]
3
+ #
4
+ compress_with Bzip2
@@ -0,0 +1,7 @@
1
+ ##
2
+ # Custom [Compressor]
3
+ #
4
+ compress_with Custom do |compressor|
5
+ compressor.command = 'gzip'
6
+ compressor.extension = '.gz'
7
+ end
@@ -0,0 +1,4 @@
1
+ ##
2
+ # Gzip [Compressor]
3
+ #
4
+ compress_with Gzip
@@ -0,0 +1,123 @@
1
+ # encoding: utf-8
2
+
3
+ ##
4
+ # Backup v<%= Backup::VERSION.split('.').first %>.x Configuration
5
+ #
6
+ # Documentation: http://backup.github.io/backup
7
+ # Issue Tracker: https://github.com/backup/backup/issues
8
+
9
+ ##
10
+ # Config Options
11
+ #
12
+ # The options here may be overridden on the command line, but the result
13
+ # will depend on the use of --root-path on the command line.
14
+ #
15
+ # If --root-path is used on the command line, then all paths set here
16
+ # will be overridden. If a path (like --tmp-path) is not given along with
17
+ # --root-path, that path will use it's default location _relative to --root-path_.
18
+ #
19
+ # If --root-path is not used on the command line, a path option (like --tmp-path)
20
+ # given on the command line will override the tmp_path set here, but all other
21
+ # paths set here will be used.
22
+ #
23
+ # Note that relative paths given on the command line without --root-path
24
+ # are relative to the current directory. The root_path set here only applies
25
+ # to relative paths set here.
26
+ #
27
+ # ---
28
+ #
29
+ # Sets the root path for all relative paths, including default paths.
30
+ # May be an absolute path, or relative to the current working directory.
31
+ #
32
+ # root_path 'my/root'
33
+ #
34
+ # Sets the path where backups are processed until they're stored.
35
+ # This must have enough free space to hold apx. 2 backups.
36
+ # May be an absolute path, or relative to the current directory or +root_path+.
37
+ #
38
+ # tmp_path 'my/tmp'
39
+ #
40
+ # Sets the path where backup stores persistent information.
41
+ # When Backup's Cycler is used, small YAML files are stored here.
42
+ # May be an absolute path, or relative to the current directory or +root_path+.
43
+ #
44
+ # data_path 'my/data'
45
+
46
+ ##
47
+ # Utilities
48
+ #
49
+ # If you need to use a utility other than the one Backup detects,
50
+ # or a utility can not be found in your $PATH.
51
+ #
52
+ # Utilities.configure do
53
+ # tar '/usr/bin/gnutar'
54
+ # redis_cli '/opt/redis/redis-cli'
55
+ # end
56
+
57
+ ##
58
+ # Logging
59
+ #
60
+ # Logging options may be set on the command line, but certain settings
61
+ # may only be configured here.
62
+ #
63
+ # Logger.configure do
64
+ # console.quiet = true # Same as command line: --quiet
65
+ # logfile.max_bytes = 2_000_000 # Default: 500_000
66
+ # syslog.enabled = true # Same as command line: --syslog
67
+ # syslog.ident = 'my_app_backup' # Default: 'backup'
68
+ # end
69
+ #
70
+ # Command line options will override those set here.
71
+ # For example, the following would override the example settings above
72
+ # to disable syslog and enable console output.
73
+ # backup perform --trigger my_backup --no-syslog --no-quiet
74
+
75
+ ##
76
+ # Component Defaults
77
+ #
78
+ # Set default options to be applied to components in all models.
79
+ # Options set within a model will override those set here.
80
+ #
81
+ # Storage::S3.defaults do |s3|
82
+ # s3.access_key_id = "my_access_key_id"
83
+ # s3.secret_access_key = "my_secret_access_key"
84
+ # end
85
+ #
86
+ # Notifier::Mail.defaults do |mail|
87
+ # mail.from = 'sender@email.com'
88
+ # mail.to = 'receiver@email.com'
89
+ # mail.address = 'smtp.gmail.com'
90
+ # mail.port = 587
91
+ # mail.domain = 'your.host.name'
92
+ # mail.user_name = 'sender@email.com'
93
+ # mail.password = 'my_password'
94
+ # mail.authentication = 'plain'
95
+ # mail.encryption = :starttls
96
+ # end
97
+
98
+ ##
99
+ # Preconfigured Models
100
+ #
101
+ # Create custom models with preconfigured components.
102
+ # Components added within the model definition will
103
+ # +add to+ the preconfigured components.
104
+ #
105
+ # preconfigure 'MyModel' do
106
+ # archive :user_pictures do |archive|
107
+ # archive.add '~/pictures'
108
+ # end
109
+ #
110
+ # notify_by Mail do |mail|
111
+ # mail.to = 'admin@email.com'
112
+ # end
113
+ # end
114
+ #
115
+ # MyModel.new(:john_smith, 'John Smith Backup') do
116
+ # archive :user_music do |archive|
117
+ # archive.add '~/music'
118
+ # end
119
+ #
120
+ # notify_by Mail do |mail|
121
+ # mail.to = 'john.smith@email.com'
122
+ # end
123
+ # end