backup 3.0.20 → 3.0.21
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +1 -5
- data/Gemfile.lock +46 -50
- data/README.md +54 -27
- data/lib/backup.rb +16 -39
- data/lib/backup/archive.rb +42 -18
- data/lib/backup/cleaner.rb +110 -25
- data/lib/backup/cli/helpers.rb +17 -32
- data/lib/backup/cli/utility.rb +46 -107
- data/lib/backup/compressor/base.rb +14 -2
- data/lib/backup/compressor/bzip2.rb +10 -24
- data/lib/backup/compressor/gzip.rb +10 -24
- data/lib/backup/compressor/lzma.rb +10 -23
- data/lib/backup/compressor/pbzip2.rb +12 -32
- data/lib/backup/config.rb +171 -0
- data/lib/backup/configuration/compressor/base.rb +1 -2
- data/lib/backup/configuration/compressor/pbzip2.rb +4 -4
- data/lib/backup/configuration/database/base.rb +2 -1
- data/lib/backup/configuration/database/mongodb.rb +8 -0
- data/lib/backup/configuration/database/mysql.rb +4 -0
- data/lib/backup/configuration/database/postgresql.rb +4 -0
- data/lib/backup/configuration/database/redis.rb +4 -0
- data/lib/backup/configuration/database/riak.rb +5 -1
- data/lib/backup/configuration/encryptor/base.rb +1 -2
- data/lib/backup/configuration/encryptor/open_ssl.rb +1 -1
- data/lib/backup/configuration/helpers.rb +7 -2
- data/lib/backup/configuration/notifier/base.rb +4 -28
- data/lib/backup/configuration/storage/base.rb +1 -1
- data/lib/backup/configuration/storage/dropbox.rb +14 -4
- data/lib/backup/configuration/syncer/base.rb +10 -0
- data/lib/backup/configuration/syncer/rsync/base.rb +28 -0
- data/lib/backup/configuration/syncer/rsync/local.rb +11 -0
- data/lib/backup/configuration/syncer/rsync/pull.rb +11 -0
- data/lib/backup/configuration/syncer/rsync/push.rb +31 -0
- data/lib/backup/configuration/syncer/s3.rb +0 -4
- data/lib/backup/database/base.rb +25 -7
- data/lib/backup/database/mongodb.rb +112 -75
- data/lib/backup/database/mysql.rb +54 -29
- data/lib/backup/database/postgresql.rb +60 -42
- data/lib/backup/database/redis.rb +61 -39
- data/lib/backup/database/riak.rb +35 -11
- data/lib/backup/dependency.rb +4 -5
- data/lib/backup/encryptor/base.rb +13 -1
- data/lib/backup/encryptor/gpg.rb +39 -39
- data/lib/backup/encryptor/open_ssl.rb +28 -38
- data/lib/backup/logger.rb +20 -11
- data/lib/backup/model.rb +206 -163
- data/lib/backup/notifier/base.rb +27 -25
- data/lib/backup/notifier/campfire.rb +7 -13
- data/lib/backup/notifier/hipchat.rb +28 -28
- data/lib/backup/notifier/mail.rb +24 -26
- data/lib/backup/notifier/presently.rb +10 -18
- data/lib/backup/notifier/prowl.rb +9 -17
- data/lib/backup/notifier/twitter.rb +11 -18
- data/lib/backup/package.rb +47 -0
- data/lib/backup/packager.rb +81 -16
- data/lib/backup/splitter.rb +48 -35
- data/lib/backup/storage/base.rb +44 -172
- data/lib/backup/storage/cloudfiles.rb +31 -46
- data/lib/backup/storage/cycler.rb +117 -0
- data/lib/backup/storage/dropbox.rb +92 -76
- data/lib/backup/storage/ftp.rb +30 -40
- data/lib/backup/storage/local.rb +44 -45
- data/lib/backup/storage/ninefold.rb +55 -49
- data/lib/backup/storage/rsync.rb +49 -56
- data/lib/backup/storage/s3.rb +33 -44
- data/lib/backup/storage/scp.rb +21 -48
- data/lib/backup/storage/sftp.rb +26 -40
- data/lib/backup/syncer/base.rb +7 -0
- data/lib/backup/syncer/rsync/base.rb +78 -0
- data/lib/backup/syncer/rsync/local.rb +53 -0
- data/lib/backup/syncer/rsync/pull.rb +38 -0
- data/lib/backup/syncer/rsync/push.rb +113 -0
- data/lib/backup/syncer/s3.rb +42 -32
- data/lib/backup/version.rb +1 -1
- data/spec/archive_spec.rb +235 -69
- data/spec/cleaner_spec.rb +304 -0
- data/spec/cli/helpers_spec.rb +142 -1
- data/spec/cli/utility_spec.rb +338 -13
- data/spec/compressor/base_spec.rb +31 -0
- data/spec/compressor/bzip2_spec.rb +60 -35
- data/spec/compressor/gzip_spec.rb +60 -35
- data/spec/compressor/lzma_spec.rb +60 -35
- data/spec/compressor/pbzip2_spec.rb +98 -37
- data/spec/config_spec.rb +321 -0
- data/spec/configuration/base_spec.rb +4 -4
- data/spec/configuration/compressor/bzip2_spec.rb +1 -0
- data/spec/configuration/compressor/gzip_spec.rb +1 -0
- data/spec/configuration/compressor/lzma_spec.rb +1 -0
- data/spec/configuration/compressor/pbzip2_spec.rb +32 -0
- data/spec/configuration/database/base_spec.rb +2 -1
- data/spec/configuration/database/mongodb_spec.rb +26 -16
- data/spec/configuration/database/mysql_spec.rb +4 -0
- data/spec/configuration/database/postgresql_spec.rb +4 -0
- data/spec/configuration/database/redis_spec.rb +4 -0
- data/spec/configuration/database/riak_spec.rb +4 -0
- data/spec/configuration/encryptor/gpg_spec.rb +1 -0
- data/spec/configuration/encryptor/open_ssl_spec.rb +1 -0
- data/spec/configuration/notifier/base_spec.rb +32 -0
- data/spec/configuration/notifier/campfire_spec.rb +1 -0
- data/spec/configuration/notifier/hipchat_spec.rb +1 -0
- data/spec/configuration/notifier/mail_spec.rb +1 -0
- data/spec/configuration/notifier/presently_spec.rb +1 -0
- data/spec/configuration/notifier/prowl_spec.rb +1 -0
- data/spec/configuration/notifier/twitter_spec.rb +1 -0
- data/spec/configuration/storage/cloudfiles_spec.rb +1 -0
- data/spec/configuration/storage/dropbox_spec.rb +4 -3
- data/spec/configuration/storage/ftp_spec.rb +1 -0
- data/spec/configuration/storage/local_spec.rb +1 -0
- data/spec/configuration/storage/ninefold_spec.rb +1 -0
- data/spec/configuration/storage/rsync_spec.rb +3 -1
- data/spec/configuration/storage/s3_spec.rb +1 -0
- data/spec/configuration/storage/scp_spec.rb +1 -0
- data/spec/configuration/storage/sftp_spec.rb +1 -0
- data/spec/configuration/syncer/rsync/base_spec.rb +33 -0
- data/spec/configuration/syncer/rsync/local_spec.rb +10 -0
- data/spec/configuration/syncer/rsync/pull_spec.rb +10 -0
- data/spec/configuration/syncer/{rsync_spec.rb → rsync/push_spec.rb} +12 -15
- data/spec/configuration/syncer/s3_spec.rb +2 -3
- data/spec/database/base_spec.rb +35 -20
- data/spec/database/mongodb_spec.rb +298 -119
- data/spec/database/mysql_spec.rb +147 -72
- data/spec/database/postgresql_spec.rb +155 -100
- data/spec/database/redis_spec.rb +200 -97
- data/spec/database/riak_spec.rb +82 -24
- data/spec/dependency_spec.rb +49 -0
- data/spec/encryptor/base_spec.rb +30 -0
- data/spec/encryptor/gpg_spec.rb +105 -28
- data/spec/encryptor/open_ssl_spec.rb +85 -114
- data/spec/logger_spec.rb +74 -8
- data/spec/model_spec.rb +528 -220
- data/spec/notifier/base_spec.rb +89 -0
- data/spec/notifier/campfire_spec.rb +147 -119
- data/spec/notifier/hipchat_spec.rb +140 -145
- data/spec/notifier/mail_spec.rb +190 -248
- data/spec/notifier/presently_spec.rb +147 -282
- data/spec/notifier/prowl_spec.rb +79 -111
- data/spec/notifier/twitter_spec.rb +87 -106
- data/spec/package_spec.rb +61 -0
- data/spec/packager_spec.rb +154 -0
- data/spec/spec_helper.rb +36 -13
- data/spec/splitter_spec.rb +90 -41
- data/spec/storage/base_spec.rb +95 -239
- data/spec/storage/cloudfiles_spec.rb +185 -75
- data/spec/storage/cycler_spec.rb +239 -0
- data/spec/storage/dropbox_spec.rb +318 -87
- data/spec/storage/ftp_spec.rb +165 -152
- data/spec/storage/local_spec.rb +206 -54
- data/spec/storage/ninefold_spec.rb +264 -128
- data/spec/storage/rsync_spec.rb +244 -163
- data/spec/storage/s3_spec.rb +175 -64
- data/spec/storage/scp_spec.rb +156 -150
- data/spec/storage/sftp_spec.rb +153 -135
- data/spec/syncer/base_spec.rb +22 -0
- data/spec/syncer/rsync/base_spec.rb +118 -0
- data/spec/syncer/rsync/local_spec.rb +121 -0
- data/spec/syncer/rsync/pull_spec.rb +90 -0
- data/spec/syncer/rsync/push_spec.rb +327 -0
- data/spec/syncer/s3_spec.rb +180 -91
- data/templates/cli/utility/config +1 -1
- data/templates/cli/utility/database/mongodb +4 -0
- data/templates/cli/utility/database/mysql +3 -0
- data/templates/cli/utility/database/postgresql +3 -0
- data/templates/cli/utility/database/redis +3 -0
- data/templates/cli/utility/database/riak +3 -0
- data/templates/cli/utility/storage/dropbox +4 -1
- data/templates/cli/utility/syncer/rsync_local +12 -0
- data/templates/cli/utility/syncer/{rsync → rsync_pull} +2 -2
- data/templates/cli/utility/syncer/rsync_push +17 -0
- data/templates/storage/dropbox/authorization_url.erb +1 -1
- metadata +42 -17
- data/lib/backup/configuration/syncer/rsync.rb +0 -45
- data/lib/backup/finder.rb +0 -87
- data/lib/backup/storage/object.rb +0 -47
- data/lib/backup/syncer/rsync.rb +0 -152
- data/spec/backup_spec.rb +0 -11
- data/spec/finder_spec.rb +0 -91
- data/spec/storage/object_spec.rb +0 -74
- data/spec/syncer/rsync_spec.rb +0 -195
data/lib/backup/dependency.rb
CHANGED
@@ -9,7 +9,6 @@ module Backup
|
|
9
9
|
# has not been installed, or when the gem's version is incorrect, and provide the
|
10
10
|
# command to install the gem. These dependencies are dynamically loaded in the Gemfile
|
11
11
|
class Dependency
|
12
|
-
extend Backup::CLI
|
13
12
|
|
14
13
|
##
|
15
14
|
# Returns a hash of dependencies that Backup requires
|
@@ -22,9 +21,9 @@ module Backup
|
|
22
21
|
:for => 'Amazon S3, Rackspace Cloud Files (S3, CloudFiles Storages)'
|
23
22
|
},
|
24
23
|
|
25
|
-
'dropbox' => {
|
26
|
-
:require => '
|
27
|
-
:version => '~> 1.
|
24
|
+
'dropbox-sdk' => {
|
25
|
+
:require => 'dropbox_sdk',
|
26
|
+
:version => '~> 1.1.0',
|
28
27
|
:for => 'Dropbox Web Service (Dropbox Storage)'
|
29
28
|
},
|
30
29
|
|
@@ -101,7 +100,7 @@ module Backup
|
|
101
100
|
> gem install #{name} -v '#{all[name][:version]}'
|
102
101
|
Please try again after installing the missing dependency.
|
103
102
|
EOS
|
104
|
-
exit
|
103
|
+
exit 1
|
105
104
|
end
|
106
105
|
end
|
107
106
|
|
@@ -6,11 +6,23 @@ module Backup
|
|
6
6
|
include Backup::CLI::Helpers
|
7
7
|
include Backup::Configuration::Helpers
|
8
8
|
|
9
|
+
def initialize
|
10
|
+
load_defaults!
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
##
|
16
|
+
# Return the encryptor name, with Backup namespace removed
|
17
|
+
def encryptor_name
|
18
|
+
self.class.to_s.sub('Backup::', '')
|
19
|
+
end
|
20
|
+
|
9
21
|
##
|
10
22
|
# Logs a message to the console and log file to inform
|
11
23
|
# the client that Backup is encrypting the archive
|
12
24
|
def log!
|
13
|
-
Logger.message "#{
|
25
|
+
Logger.message "Using #{ encryptor_name } to encrypt the archive."
|
14
26
|
end
|
15
27
|
end
|
16
28
|
end
|
data/lib/backup/encryptor/gpg.rb
CHANGED
@@ -1,9 +1,5 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
##
|
4
|
-
# Require the tempfile Ruby library when Backup::Encryptor::GPG is loaded
|
5
|
-
require 'tempfile'
|
6
|
-
|
7
3
|
module Backup
|
8
4
|
module Encryptor
|
9
5
|
class GPG < Base
|
@@ -12,67 +8,71 @@ module Backup
|
|
12
8
|
# The GPG Public key that'll be used to encrypt the backup
|
13
9
|
attr_accessor :key
|
14
10
|
|
15
|
-
##
|
16
|
-
# Contains the GPG encryption key id which'll be extracted from the public key file
|
17
|
-
attr_accessor :encryption_key_id
|
18
|
-
|
19
|
-
##
|
20
|
-
# Contains the temporary file with the public key
|
21
|
-
attr_accessor :tmp_file
|
22
|
-
|
23
11
|
##
|
24
12
|
# Creates a new instance of Backup::Encryptor::GPG and
|
25
13
|
# sets the key to the provided GPG key. To enhance the DSL
|
26
14
|
# the user may use tabs and spaces to indent the multi-line key string
|
27
15
|
# since we gsub() every preceding 'space' and 'tab' on each line
|
28
16
|
def initialize(&block)
|
29
|
-
|
17
|
+
super
|
30
18
|
|
31
19
|
instance_eval(&block) if block_given?
|
32
|
-
|
33
|
-
@key = key.gsub(/^[[:blank:]]+/, '')
|
34
20
|
end
|
35
21
|
|
36
22
|
##
|
37
|
-
#
|
38
|
-
#
|
39
|
-
|
23
|
+
# This is called as part of the procedure run by the Packager.
|
24
|
+
# It sets up the needed encryption_key_email to pass to the gpg command,
|
25
|
+
# then yields the command to use as part of the packaging procedure.
|
26
|
+
# Once the packaging procedure is complete, it will return
|
27
|
+
# so that any clean-up may be performed after the yield.
|
28
|
+
def encrypt_with
|
40
29
|
log!
|
41
|
-
|
42
|
-
extract_encryption_key_id!
|
30
|
+
extract_encryption_key_email!
|
43
31
|
|
44
|
-
|
32
|
+
yield "#{ utility(:gpg) } #{ options }", '.gpg'
|
33
|
+
end
|
45
34
|
|
46
|
-
|
47
|
-
tmp_file.unlink
|
35
|
+
private
|
48
36
|
|
49
|
-
|
37
|
+
##
|
38
|
+
# Imports the given encryption key to ensure it's available for use,
|
39
|
+
# and extracts the email address used to create the key.
|
40
|
+
# This is stored in '@encryption_key_email', to be used to specify
|
41
|
+
# the --recipient when performing encryption so this key is used.
|
42
|
+
def extract_encryption_key_email!
|
43
|
+
if @encryption_key_email.to_s.empty?
|
44
|
+
with_tmp_key_file do |tmp_file|
|
45
|
+
@encryption_key_email = run(
|
46
|
+
"#{ utility(:gpg) } --import '#{tmp_file}' 2>&1"
|
47
|
+
).match(/<(.+)>/)[1]
|
48
|
+
end
|
49
|
+
end
|
50
50
|
end
|
51
51
|
|
52
|
-
private
|
53
|
-
|
54
52
|
##
|
55
53
|
# GPG options
|
56
|
-
# Sets the gpg mode to 'encrypt' and passes in the
|
54
|
+
# Sets the gpg mode to 'encrypt' and passes in the encryption_key_email
|
57
55
|
def options
|
58
|
-
"-e --trust-model always -r '#{
|
56
|
+
"-e --trust-model always -r '#{ @encryption_key_email }'"
|
59
57
|
end
|
60
58
|
|
61
59
|
##
|
62
|
-
#
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
FileUtils.
|
67
|
-
|
68
|
-
|
60
|
+
# Writes the provided public gpg key to a temp file,
|
61
|
+
# yields the path, then deletes the file when the block returns.
|
62
|
+
def with_tmp_key_file
|
63
|
+
tmp_file = Tempfile.new('backup.pub')
|
64
|
+
FileUtils.chown(Config.user, nil, tmp_file.path)
|
65
|
+
FileUtils.chmod(0600, tmp_file.path)
|
66
|
+
tmp_file.write(encryption_key)
|
67
|
+
tmp_file.close
|
68
|
+
yield tmp_file.path
|
69
|
+
tmp_file.delete
|
69
70
|
end
|
70
71
|
|
71
72
|
##
|
72
|
-
#
|
73
|
-
|
74
|
-
|
75
|
-
@encryption_key_id = run("#{ utility(:gpg) } --import '#{tmp_file.path}' 2>&1").match(/<(.+)>/)[1]
|
73
|
+
# Returns the encryption key with preceding spaces and tabs removed
|
74
|
+
def encryption_key
|
75
|
+
key.gsub(/^[[:blank:]]+/, '')
|
76
76
|
end
|
77
77
|
|
78
78
|
end
|
@@ -15,66 +15,56 @@ module Backup
|
|
15
15
|
|
16
16
|
##
|
17
17
|
# Determines whether the 'base64' should be used or not
|
18
|
-
|
18
|
+
attr_accessor :base64
|
19
19
|
|
20
20
|
##
|
21
21
|
# Determines whether the 'salt' flag should be used
|
22
|
-
|
22
|
+
attr_accessor :salt
|
23
23
|
|
24
24
|
##
|
25
25
|
# Creates a new instance of Backup::Encryptor::OpenSSL and
|
26
26
|
# sets the password attribute to what was provided
|
27
27
|
def initialize(&block)
|
28
|
-
|
28
|
+
super
|
29
29
|
|
30
30
|
@base64 ||= false
|
31
|
-
@salt ||=
|
31
|
+
@salt ||= true
|
32
32
|
@password_file ||= nil
|
33
33
|
|
34
34
|
instance_eval(&block) if block_given?
|
35
35
|
end
|
36
36
|
|
37
37
|
##
|
38
|
-
#
|
39
|
-
|
38
|
+
# This is called as part of the procedure run by the Packager.
|
39
|
+
# It sets up the needed options to pass to the openssl command,
|
40
|
+
# then yields the command to use as part of the packaging procedure.
|
41
|
+
# Once the packaging procedure is complete, it will return
|
42
|
+
# so that any clean-up may be performed after the yield.
|
43
|
+
def encrypt_with
|
40
44
|
log!
|
41
|
-
|
42
|
-
rm(Backup::Model.file)
|
43
|
-
Backup::Model.extension += '.enc'
|
45
|
+
yield "#{ utility(:openssl) } #{ options }", '.enc'
|
44
46
|
end
|
45
47
|
|
46
|
-
|
48
|
+
private
|
47
49
|
|
48
50
|
##
|
49
|
-
#
|
50
|
-
#
|
51
|
-
|
52
|
-
|
53
|
-
end
|
54
|
-
|
55
|
-
##
|
56
|
-
# Returns '-base64' if @base64 is set to 'true'.
|
57
|
-
# This option will make the encrypted output base64 encoded,
|
51
|
+
# Uses the 256bit AES encryption cipher, which is what the
|
52
|
+
# US Government uses to encrypt information at the "Top Secret" level.
|
53
|
+
#
|
54
|
+
# The -base64 option will make the encrypted output base64 encoded,
|
58
55
|
# this makes the encrypted file readable using text editors
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
# Returns '-pass file:<password file>' when @password_file has been set.
|
72
|
-
def pass
|
73
|
-
if @password_file
|
74
|
-
["-pass file:#{@password_file}"]
|
75
|
-
else
|
76
|
-
["-k '#{@password}'"]
|
77
|
-
end
|
56
|
+
#
|
57
|
+
# The -salt option adds strength to the encryption
|
58
|
+
#
|
59
|
+
# Always sets a password option, if even no password is given,
|
60
|
+
# but will prefer the password_file option if both are given.
|
61
|
+
def options
|
62
|
+
opts = ['aes-256-cbc']
|
63
|
+
opts << '-base64' if @base64
|
64
|
+
opts << '-salt' if @salt
|
65
|
+
opts << ( @password_file.to_s.empty? ?
|
66
|
+
"-k '#{@password}'" : "-pass file:#{@password_file}" )
|
67
|
+
opts.join(' ')
|
78
68
|
end
|
79
69
|
|
80
70
|
end
|
data/lib/backup/logger.rb
CHANGED
@@ -4,6 +4,8 @@ module Backup
|
|
4
4
|
module Logger
|
5
5
|
class << self
|
6
6
|
|
7
|
+
attr_accessor :quiet
|
8
|
+
|
7
9
|
##
|
8
10
|
# Outputs a messages to the console and writes it to the backup.log
|
9
11
|
def message(string)
|
@@ -58,6 +60,22 @@ module Backup
|
|
58
60
|
@has_warnings = false
|
59
61
|
end
|
60
62
|
|
63
|
+
def truncate!(max_bytes = 500_000)
|
64
|
+
log_file = File.join(Config.log_path, 'backup.log')
|
65
|
+
if File.stat(log_file).size > max_bytes
|
66
|
+
FileUtils.mv(log_file, log_file + '~')
|
67
|
+
File.open(log_file + '~', 'r') do |io_in|
|
68
|
+
File.open(log_file, 'w') do |io_out|
|
69
|
+
io_in.seek(-max_bytes, IO::SEEK_END) && io_in.gets
|
70
|
+
while line = io_in.gets
|
71
|
+
io_out.puts line
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
FileUtils.rm_f(log_file + '~')
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
61
79
|
private
|
62
80
|
|
63
81
|
##
|
@@ -86,14 +104,14 @@ module Backup
|
|
86
104
|
##
|
87
105
|
# Receives an Array of Strings to be written to the console.
|
88
106
|
def to_console(lines, stderr = false)
|
89
|
-
return if quiet
|
107
|
+
return if quiet
|
90
108
|
lines.each {|line| stderr ? Kernel.warn(line) : puts(line) }
|
91
109
|
end
|
92
110
|
|
93
111
|
##
|
94
112
|
# Receives an Array of Strings to be written to the log file.
|
95
113
|
def to_file(lines)
|
96
|
-
File.open(File.join(
|
114
|
+
File.open(File.join(Config.log_path, 'backup.log'), 'a') do |file|
|
97
115
|
lines.each {|line| file.puts line }
|
98
116
|
end
|
99
117
|
messages.push(*lines)
|
@@ -127,15 +145,6 @@ module Backup
|
|
127
145
|
"\e[#{code}m#{string}\e[0m"
|
128
146
|
end
|
129
147
|
|
130
|
-
##
|
131
|
-
# Returns 'true' (boolean) if the QUIET constant is defined
|
132
|
-
# By default it isn't defined, only when initializing Backup using
|
133
|
-
# the '--quiet' (or '-q') option in the CLI
|
134
|
-
# (e.g. backup perform -t my_backup --quiet)
|
135
|
-
def quiet?
|
136
|
-
const_defined?(:QUIET) && QUIET
|
137
|
-
end
|
138
|
-
|
139
148
|
end
|
140
149
|
end
|
141
150
|
end
|
data/lib/backup/model.rb
CHANGED
@@ -4,196 +4,186 @@ module Backup
|
|
4
4
|
class Model
|
5
5
|
include Backup::CLI::Helpers
|
6
6
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
7
|
+
class << self
|
8
|
+
##
|
9
|
+
# The Backup::Model.all class method keeps track of all the models
|
10
|
+
# that have been instantiated. It returns the @all class variable,
|
11
|
+
# which contains an array of all the models
|
12
|
+
def all
|
13
|
+
@all ||= []
|
14
|
+
end
|
11
15
|
|
12
|
-
|
13
|
-
|
14
|
-
|
16
|
+
##
|
17
|
+
# Return the first model matching +trigger+.
|
18
|
+
# Raises Errors::MissingTriggerError if no matches are found.
|
19
|
+
def find(trigger)
|
20
|
+
trigger = trigger.to_s
|
21
|
+
all.each do |model|
|
22
|
+
return model if model.trigger == trigger
|
23
|
+
end
|
24
|
+
raise Errors::Model::MissingTriggerError,
|
25
|
+
"Could not find trigger '#{trigger}'."
|
26
|
+
end
|
27
|
+
|
28
|
+
##
|
29
|
+
# Find and return an Array of all models matching +trigger+
|
30
|
+
# Used to match triggers using a wildcard (*)
|
31
|
+
def find_matching(trigger)
|
32
|
+
regex = /^#{ trigger.to_s.gsub('*', '(.*)') }$/
|
33
|
+
all.select {|model| regex =~ model.trigger }
|
34
|
+
end
|
35
|
+
end
|
15
36
|
|
16
37
|
##
|
17
|
-
# The
|
18
|
-
|
38
|
+
# The trigger (stored as a String) is used as an identifier
|
39
|
+
# for initializing the backup process
|
40
|
+
attr_reader :trigger
|
19
41
|
|
20
42
|
##
|
21
|
-
# The
|
22
|
-
|
43
|
+
# The label (stored as a String) is used for a more friendly user output
|
44
|
+
attr_reader :label
|
23
45
|
|
24
46
|
##
|
25
|
-
# The
|
26
|
-
|
47
|
+
# The databases attribute holds an array of database objects
|
48
|
+
attr_reader :databases
|
27
49
|
|
28
50
|
##
|
29
|
-
# The
|
30
|
-
|
51
|
+
# The archives attr_accessor holds an array of archive objects
|
52
|
+
attr_reader :archives
|
31
53
|
|
32
54
|
##
|
33
55
|
# The notifiers attr_accessor holds an array of notifier objects
|
34
|
-
|
56
|
+
attr_reader :notifiers
|
35
57
|
|
36
58
|
##
|
37
59
|
# The storages attribute holds an array of storage objects
|
38
|
-
|
60
|
+
attr_reader :storages
|
39
61
|
|
40
62
|
##
|
41
63
|
# The syncers attribute holds an array of syncer objects
|
42
|
-
|
64
|
+
attr_reader :syncers
|
43
65
|
|
44
66
|
##
|
45
|
-
#
|
46
|
-
|
67
|
+
# Holds the configured Compressor
|
68
|
+
attr_reader :compressor
|
47
69
|
|
48
70
|
##
|
49
|
-
#
|
50
|
-
|
51
|
-
|
52
|
-
class << self
|
53
|
-
##
|
54
|
-
# The Backup::Model.all class method keeps track of all the models
|
55
|
-
# that have been instantiated. It returns the @all class variable,
|
56
|
-
# which contains an array of all the models
|
57
|
-
attr_accessor :all
|
58
|
-
|
59
|
-
##
|
60
|
-
# Contains the current file extension (this changes from time to time after a file
|
61
|
-
# gets compressed or encrypted so we can keep track of the correct file when new
|
62
|
-
# extensions get appended to the current file name)
|
63
|
-
attr_accessor :extension
|
64
|
-
|
65
|
-
##
|
66
|
-
# Contains the currently-in-use model. This attribute should get set by Backup::Finder.
|
67
|
-
# Use Backup::Model.current to retrieve the actual data of the model
|
68
|
-
attr_accessor :current
|
69
|
-
|
70
|
-
##
|
71
|
-
# Contains an array of chunk suffixes for a given file
|
72
|
-
attr_accessor :chunk_suffixes
|
73
|
-
|
74
|
-
##
|
75
|
-
# Returns the full path to the current file (including the current extension).
|
76
|
-
# To just return the filename and extension without the path, use File.basename(Backup::Model.file)
|
77
|
-
def file
|
78
|
-
File.join(TMP_PATH, "#{ TIME }.#{ TRIGGER }.#{ Backup::Model.extension }")
|
79
|
-
end
|
80
|
-
|
81
|
-
##
|
82
|
-
# Returns the @chunk_suffixes variable, sets it to an emtpy array if nil
|
83
|
-
def chunk_suffixes
|
84
|
-
@chunk_suffixes ||= Array.new
|
85
|
-
end
|
71
|
+
# Holds the configured Encryptor
|
72
|
+
attr_reader :encryptor
|
86
73
|
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
def tmp_path
|
91
|
-
File.join(TMP_PATH, TRIGGER)
|
92
|
-
end
|
93
|
-
end
|
74
|
+
##
|
75
|
+
# Holds the configured Splitter
|
76
|
+
attr_reader :splitter
|
94
77
|
|
95
78
|
##
|
96
|
-
#
|
97
|
-
|
98
|
-
@all = Array.new
|
79
|
+
# The final backup Package this model will create.
|
80
|
+
attr_reader :package
|
99
81
|
|
100
82
|
##
|
101
|
-
#
|
102
|
-
|
83
|
+
# The time when the backup initiated (in format: 2011.02.20.03.29.59)
|
84
|
+
attr_reader :time
|
103
85
|
|
104
86
|
##
|
105
|
-
# Takes a trigger, label and the configuration block
|
106
|
-
#
|
107
|
-
#
|
108
|
-
# configuration for the model, it will append the newly created "model" instance
|
109
|
-
# to the @all class variable (Array) so it can be accessed by Backup::Finder
|
110
|
-
# and any other location
|
87
|
+
# Takes a trigger, label and the configuration block.
|
88
|
+
# After the instance has evaluated the configuration block
|
89
|
+
# to configure the model, it will be appended to Model.all
|
111
90
|
def initialize(trigger, label, &block)
|
112
|
-
@trigger
|
113
|
-
@label
|
114
|
-
@time = TIME
|
91
|
+
@trigger = trigger.to_s
|
92
|
+
@label = label.to_s
|
115
93
|
|
116
94
|
procedure_instance_variables.each do |variable|
|
117
95
|
instance_variable_set(variable, Array.new)
|
118
96
|
end
|
119
97
|
|
120
|
-
instance_eval(&block)
|
121
|
-
|
122
|
-
end
|
123
|
-
|
124
|
-
##
|
125
|
-
# Adds a database to the array of databases
|
126
|
-
# to dump during the backup process
|
127
|
-
def database(database, &block)
|
128
|
-
@databases << Backup::Database.const_get(
|
129
|
-
last_constant(database)
|
130
|
-
).new(&block)
|
98
|
+
instance_eval(&block) if block_given?
|
99
|
+
Model.all << self
|
131
100
|
end
|
132
101
|
|
133
102
|
##
|
134
103
|
# Adds an archive to the array of archives
|
135
104
|
# to store during the backup process
|
136
105
|
def archive(name, &block)
|
137
|
-
@archives <<
|
106
|
+
@archives << Archive.new(self, name, &block)
|
138
107
|
end
|
139
108
|
|
140
109
|
##
|
141
|
-
# Adds
|
142
|
-
# to
|
143
|
-
def
|
144
|
-
@
|
145
|
-
last_constant(name)
|
146
|
-
).new(&block)
|
110
|
+
# Adds a database to the array of databases
|
111
|
+
# to dump during the backup process
|
112
|
+
def database(name, &block)
|
113
|
+
@databases << get_class_from_scope(Database, name).new(self, &block)
|
147
114
|
end
|
148
115
|
|
149
116
|
##
|
150
|
-
# Adds a
|
151
|
-
# to use during the backup process
|
152
|
-
def
|
153
|
-
@
|
154
|
-
|
155
|
-
|
117
|
+
# Adds a storage method to the array of storage
|
118
|
+
# methods to use during the backup process
|
119
|
+
def store_with(name, storage_id = nil, &block)
|
120
|
+
@storages << get_class_from_scope(Storage, name).new(self, storage_id, &block)
|
121
|
+
end
|
122
|
+
|
123
|
+
##
|
124
|
+
# Adds a syncer method to the array of syncer
|
125
|
+
# methods to use during the backup process
|
126
|
+
def sync_with(name, &block)
|
127
|
+
##
|
128
|
+
# Warn user of DSL change from 'RSync' to 'RSync::Local'
|
129
|
+
if name.to_s == 'Backup::Config::RSync'
|
130
|
+
Logger.warn Errors::ConfigError.new(<<-EOS)
|
131
|
+
Configuration Update Needed for Syncer::RSync
|
132
|
+
The RSync Syncer has been split into three separate modules:
|
133
|
+
RSync::Local, RSync::Push and RSync::Pull
|
134
|
+
Please update your configuration for your local RSync Syncer
|
135
|
+
from 'sync_with RSync do ...' to 'sync_with RSync::Local do ...'
|
136
|
+
EOS
|
137
|
+
name = Backup::Config::RSync::Local
|
138
|
+
end
|
139
|
+
@syncers << get_class_from_scope(Syncer, name).new(&block)
|
156
140
|
end
|
157
141
|
|
158
142
|
##
|
159
143
|
# Adds a notifier to the array of notifiers
|
160
144
|
# to use during the backup process
|
161
145
|
def notify_by(name, &block)
|
162
|
-
@notifiers <<
|
163
|
-
last_constant(name)
|
164
|
-
).new(&block)
|
146
|
+
@notifiers << get_class_from_scope(Notifier, name).new(self, &block)
|
165
147
|
end
|
166
148
|
|
167
149
|
##
|
168
|
-
# Adds
|
169
|
-
|
170
|
-
|
171
|
-
@storages << Backup::Storage.const_get(
|
172
|
-
last_constant(storage)
|
173
|
-
).new(storage_id, &block)
|
150
|
+
# Adds an encryptor to use during the backup process
|
151
|
+
def encrypt_with(name, &block)
|
152
|
+
@encryptor = get_class_from_scope(Encryptor, name).new(&block)
|
174
153
|
end
|
175
154
|
|
176
155
|
##
|
177
|
-
# Adds a
|
178
|
-
|
179
|
-
|
180
|
-
@syncers << Backup::Syncer.const_get(
|
181
|
-
last_constant(syncer)
|
182
|
-
).new(&block)
|
156
|
+
# Adds a compressor to use during the backup process
|
157
|
+
def compress_with(name, &block)
|
158
|
+
@compressor = get_class_from_scope(Compressor, name).new(&block)
|
183
159
|
end
|
184
160
|
|
185
161
|
##
|
186
|
-
# Adds a method that allows the user to
|
187
|
-
#
|
188
|
-
#
|
189
|
-
|
190
|
-
|
162
|
+
# Adds a method that allows the user to configure this backup model
|
163
|
+
# to use a Splitter, with the given +chunk_size+
|
164
|
+
# The +chunk_size+ (in megabytes) will later determine
|
165
|
+
# in how many chunks the backup needs to be split into
|
166
|
+
def split_into_chunks_of(chunk_size)
|
167
|
+
if chunk_size.is_a?(Integer)
|
168
|
+
@splitter = Splitter.new(self, chunk_size)
|
169
|
+
else
|
170
|
+
raise Errors::Model::ConfigurationError, <<-EOS
|
171
|
+
Invalid Chunk Size for Splitter
|
172
|
+
Argument to #split_into_chunks_of() must be an Integer
|
173
|
+
EOS
|
174
|
+
end
|
191
175
|
end
|
192
176
|
|
193
177
|
##
|
194
|
-
#
|
195
|
-
|
196
|
-
|
178
|
+
# Ensure DATA_PATH and DATA_PATH/TRIGGER are created
|
179
|
+
# if they do not yet exist
|
180
|
+
#
|
181
|
+
# Clean any temporary files and/or package files left over
|
182
|
+
# from the last time this model/trigger was performed.
|
183
|
+
# Logs warnings if files exist and are cleaned.
|
184
|
+
def prepare!
|
185
|
+
FileUtils.mkdir_p(File.join(Config.data_path, trigger))
|
186
|
+
Cleaner.prepare(self)
|
197
187
|
end
|
198
188
|
|
199
189
|
##
|
@@ -211,10 +201,10 @@ module Backup
|
|
211
201
|
# the folder, it'll make a single .tar package (archive) out of it
|
212
202
|
##
|
213
203
|
# [Encryption]
|
214
|
-
# Optionally encrypts the packaged file with
|
204
|
+
# Optionally encrypts the packaged file with the configured encryptor
|
215
205
|
##
|
216
206
|
# [Compression]
|
217
|
-
# Optionally compresses the
|
207
|
+
# Optionally compresses the each Archive and Database dump with the configured compressor
|
218
208
|
##
|
219
209
|
# [Splitting]
|
220
210
|
# Optionally splits the backup file in to multiple smaller chunks before transferring them
|
@@ -234,11 +224,21 @@ module Backup
|
|
234
224
|
# any errors.
|
235
225
|
##
|
236
226
|
# [Cleaning]
|
237
|
-
#
|
238
|
-
#
|
239
|
-
#
|
240
|
-
#
|
227
|
+
# Once the final Packaging is complete, the temporary folder used will be removed.
|
228
|
+
# Then, once all Storages have run, the final packaged files will be removed.
|
229
|
+
# If any errors occur during the backup process, all temporary files will be left in place.
|
230
|
+
# If the error occurs before Packaging, then the temporary folder (tmp_path/trigger)
|
231
|
+
# will remain and may contain all or some of the configured Archives and/or Database dumps.
|
232
|
+
# If the error occurs after Packaging, but before the Storages complete, then the final
|
233
|
+
# packaged files (located in the root of tmp_path) will remain.
|
234
|
+
# *** Important *** If an error occurs and any of the above mentioned temporary files remain,
|
235
|
+
# those files *** will be removed *** before the next scheduled backup for the same trigger.
|
236
|
+
#
|
241
237
|
def perform!
|
238
|
+
@started_at = Time.now
|
239
|
+
@time = @started_at.strftime("%Y.%m.%d.%H.%M.%S")
|
240
|
+
log!(:started)
|
241
|
+
|
242
242
|
if databases.any? or archives.any?
|
243
243
|
procedures.each do |procedure|
|
244
244
|
(procedure.call; next) if procedure.is_a?(Proc)
|
@@ -247,27 +247,29 @@ module Backup
|
|
247
247
|
end
|
248
248
|
|
249
249
|
syncers.each(&:perform!)
|
250
|
-
notifiers.each
|
250
|
+
notifiers.each(&:perform!)
|
251
|
+
log!(:finished)
|
251
252
|
|
252
253
|
rescue Exception => err
|
253
254
|
fatal = !err.is_a?(StandardError)
|
254
255
|
|
255
|
-
|
256
|
+
err = Errors::ModelError.wrap(err, <<-EOS)
|
256
257
|
Backup for #{label} (#{trigger}) Failed!
|
257
258
|
An Error occured which has caused this Backup to abort before completion.
|
258
|
-
Please review the Log for this Backup to determine if steps need to be taken
|
259
|
-
to clean up, based on the point at which the failure occured.
|
260
259
|
EOS
|
261
|
-
Logger.error
|
260
|
+
Logger.error err
|
261
|
+
Logger.error "\nBacktrace:\n\s\s" + err.backtrace.join("\n\s\s") + "\n\n"
|
262
|
+
|
263
|
+
Cleaner.warnings(self)
|
262
264
|
|
263
265
|
if fatal
|
264
|
-
Logger.error
|
266
|
+
Logger.error Errors::ModelError.new(<<-EOS)
|
265
267
|
This Error was Fatal and Backup will now exit.
|
266
268
|
If you have other Backup jobs (triggers) configured to run,
|
267
269
|
they will not be processed.
|
268
270
|
EOS
|
269
271
|
else
|
270
|
-
Logger.message
|
272
|
+
Logger.message Errors::ModelError.new(<<-EOS)
|
271
273
|
If you have other Backup jobs (triggers) configured to run,
|
272
274
|
Backup will now attempt to continue...
|
273
275
|
EOS
|
@@ -275,58 +277,99 @@ module Backup
|
|
275
277
|
|
276
278
|
notifiers.each do |n|
|
277
279
|
begin
|
278
|
-
n.perform!(
|
280
|
+
n.perform!(true)
|
279
281
|
rescue Exception; end
|
280
282
|
end
|
281
283
|
|
282
284
|
exit(1) if fatal
|
283
|
-
ensure
|
284
|
-
clean!
|
285
285
|
end
|
286
286
|
|
287
|
-
|
287
|
+
private
|
288
288
|
|
289
289
|
##
|
290
290
|
# After all the databases and archives have been dumped and sorted,
|
291
|
-
# these files will be bundled in to a .tar archive (uncompressed)
|
292
|
-
#
|
291
|
+
# these files will be bundled in to a .tar archive (uncompressed),
|
292
|
+
# which may be optionally Encrypted and/or Split into multiple "chunks".
|
293
|
+
# All information about this final archive is stored in the @package.
|
294
|
+
# Once complete, the temporary folder used during packaging is removed.
|
293
295
|
def package!
|
294
|
-
|
296
|
+
@package = Package.new(self)
|
297
|
+
Packager.package!(self)
|
298
|
+
Cleaner.remove_packaging(self)
|
295
299
|
end
|
296
300
|
|
297
301
|
##
|
298
|
-
#
|
299
|
-
# passing it the current model instance and runs it.
|
300
|
-
def split!
|
301
|
-
Backup::Splitter.new(self).split!
|
302
|
-
end
|
303
|
-
|
304
|
-
##
|
305
|
-
# Cleans up the temporary files that were created after the backup process finishes
|
302
|
+
# Removes the final package file(s) once all configured Storages have run.
|
306
303
|
def clean!
|
307
|
-
|
304
|
+
Cleaner.remove_package(@package)
|
308
305
|
end
|
309
306
|
|
310
307
|
##
|
311
308
|
# Returns an array of procedures
|
312
309
|
def procedures
|
313
|
-
|
314
|
-
databases, archives, lambda { package! }, compressors,
|
315
|
-
encryptors, lambda { split! }, storages
|
316
|
-
])
|
310
|
+
[databases, archives, lambda { package! }, storages, lambda { clean! }]
|
317
311
|
end
|
318
312
|
|
319
313
|
##
|
320
314
|
# Returns an Array of the names (String) of the procedure instance variables
|
321
315
|
def procedure_instance_variables
|
322
|
-
[:@databases, :@archives, :@
|
316
|
+
[:@databases, :@archives, :@storages, :@notifiers, :@syncers]
|
317
|
+
end
|
318
|
+
|
319
|
+
##
|
320
|
+
# Returns the class/model specified by +name+ inside of +scope+.
|
321
|
+
# +scope+ should be a Class/Module.
|
322
|
+
# +name+ may be Class/Module or String representation
|
323
|
+
# of any namespace which exists under +scope+.
|
324
|
+
#
|
325
|
+
# The 'Backup::Config::' namespace is stripped from +name+,
|
326
|
+
# since this is the namespace where we define module namespaces
|
327
|
+
# for use with Model's DSL methods.
|
328
|
+
#
|
329
|
+
# Examples:
|
330
|
+
# get_class_from_scope(Backup::Database, 'MySQL')
|
331
|
+
# returns the class Backup::Database::MySQL
|
332
|
+
#
|
333
|
+
# get_class_from_scope(Backup::Syncer, Backup::Config::RSync::Local)
|
334
|
+
# returns the class Backup::Syncer::RSync::Local
|
335
|
+
#
|
336
|
+
def get_class_from_scope(scope, name)
|
337
|
+
klass = scope
|
338
|
+
name = name.to_s.sub(/^Backup::Config::/, '')
|
339
|
+
name.split('::').each do |chunk|
|
340
|
+
klass = klass.const_get(chunk)
|
341
|
+
end
|
342
|
+
klass
|
343
|
+
end
|
344
|
+
|
345
|
+
##
|
346
|
+
# Logs messages when the backup starts and finishes
|
347
|
+
def log!(action)
|
348
|
+
case action
|
349
|
+
when :started
|
350
|
+
Logger.message "Performing Backup for '#{label} (#{trigger})'!\n" +
|
351
|
+
"[ backup #{ Version.current } : #{ RUBY_DESCRIPTION } ]"
|
352
|
+
|
353
|
+
when :finished
|
354
|
+
msg = "Backup for '#{ label } (#{ trigger })' " +
|
355
|
+
"Completed %s in #{ elapsed_time }"
|
356
|
+
if Logger.has_warnings?
|
357
|
+
Logger.warn msg % 'Successfully (with Warnings)'
|
358
|
+
else
|
359
|
+
Logger.message msg % 'Successfully'
|
360
|
+
end
|
361
|
+
end
|
323
362
|
end
|
324
363
|
|
325
364
|
##
|
326
|
-
# Returns
|
327
|
-
|
328
|
-
|
329
|
-
|
365
|
+
# Returns a string representing the elapsed time since the backup started.
|
366
|
+
def elapsed_time
|
367
|
+
duration = Time.now.to_i - @started_at.to_i
|
368
|
+
hours = duration / 3600
|
369
|
+
remainder = duration - (hours * 3600)
|
370
|
+
minutes = remainder / 60
|
371
|
+
seconds = remainder - (minutes * 60)
|
372
|
+
'%02d:%02d:%02d' % [hours, minutes, seconds]
|
330
373
|
end
|
331
374
|
|
332
375
|
end
|