backup-agoddard 3.0.27
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of backup-agoddard might be problematic. Click here for more details.
- data/.gitignore +8 -0
- data/.travis.yml +10 -0
- data/Gemfile +28 -0
- data/Guardfile +23 -0
- data/LICENSE.md +24 -0
- data/README.md +478 -0
- data/backup.gemspec +32 -0
- data/bin/backup +11 -0
- data/lib/backup.rb +133 -0
- data/lib/backup/archive.rb +117 -0
- data/lib/backup/binder.rb +22 -0
- data/lib/backup/cleaner.rb +121 -0
- data/lib/backup/cli/helpers.rb +93 -0
- data/lib/backup/cli/utility.rb +255 -0
- data/lib/backup/compressor/base.rb +35 -0
- data/lib/backup/compressor/bzip2.rb +50 -0
- data/lib/backup/compressor/custom.rb +53 -0
- data/lib/backup/compressor/gzip.rb +50 -0
- data/lib/backup/compressor/lzma.rb +52 -0
- data/lib/backup/compressor/pbzip2.rb +59 -0
- data/lib/backup/config.rb +174 -0
- data/lib/backup/configuration.rb +33 -0
- data/lib/backup/configuration/helpers.rb +130 -0
- data/lib/backup/configuration/store.rb +24 -0
- data/lib/backup/database/base.rb +53 -0
- data/lib/backup/database/mongodb.rb +230 -0
- data/lib/backup/database/mysql.rb +160 -0
- data/lib/backup/database/postgresql.rb +144 -0
- data/lib/backup/database/redis.rb +136 -0
- data/lib/backup/database/riak.rb +67 -0
- data/lib/backup/dependency.rb +108 -0
- data/lib/backup/encryptor/base.rb +29 -0
- data/lib/backup/encryptor/gpg.rb +760 -0
- data/lib/backup/encryptor/open_ssl.rb +72 -0
- data/lib/backup/errors.rb +124 -0
- data/lib/backup/hooks.rb +68 -0
- data/lib/backup/logger.rb +152 -0
- data/lib/backup/model.rb +409 -0
- data/lib/backup/notifier/base.rb +81 -0
- data/lib/backup/notifier/campfire.rb +155 -0
- data/lib/backup/notifier/hipchat.rb +93 -0
- data/lib/backup/notifier/mail.rb +206 -0
- data/lib/backup/notifier/prowl.rb +65 -0
- data/lib/backup/notifier/pushover.rb +88 -0
- data/lib/backup/notifier/twitter.rb +70 -0
- data/lib/backup/package.rb +47 -0
- data/lib/backup/packager.rb +100 -0
- data/lib/backup/pipeline.rb +110 -0
- data/lib/backup/splitter.rb +75 -0
- data/lib/backup/storage/base.rb +99 -0
- data/lib/backup/storage/cloudfiles.rb +87 -0
- data/lib/backup/storage/cycler.rb +117 -0
- data/lib/backup/storage/dropbox.rb +178 -0
- data/lib/backup/storage/ftp.rb +119 -0
- data/lib/backup/storage/local.rb +82 -0
- data/lib/backup/storage/ninefold.rb +116 -0
- data/lib/backup/storage/rsync.rb +149 -0
- data/lib/backup/storage/s3.rb +94 -0
- data/lib/backup/storage/scp.rb +99 -0
- data/lib/backup/storage/sftp.rb +108 -0
- data/lib/backup/syncer/base.rb +46 -0
- data/lib/backup/syncer/cloud/base.rb +247 -0
- data/lib/backup/syncer/cloud/cloud_files.rb +78 -0
- data/lib/backup/syncer/cloud/s3.rb +68 -0
- data/lib/backup/syncer/rsync/base.rb +49 -0
- data/lib/backup/syncer/rsync/local.rb +55 -0
- data/lib/backup/syncer/rsync/pull.rb +36 -0
- data/lib/backup/syncer/rsync/push.rb +116 -0
- data/lib/backup/template.rb +46 -0
- data/lib/backup/version.rb +43 -0
- data/spec-live/.gitignore +6 -0
- data/spec-live/README +7 -0
- data/spec-live/backups/config.rb +83 -0
- data/spec-live/backups/config.yml.template +46 -0
- data/spec-live/backups/models.rb +184 -0
- data/spec-live/compressor/custom_spec.rb +30 -0
- data/spec-live/compressor/gzip_spec.rb +30 -0
- data/spec-live/encryptor/gpg_keys.rb +239 -0
- data/spec-live/encryptor/gpg_spec.rb +287 -0
- data/spec-live/notifier/mail_spec.rb +121 -0
- data/spec-live/spec_helper.rb +151 -0
- data/spec-live/storage/dropbox_spec.rb +151 -0
- data/spec-live/storage/local_spec.rb +83 -0
- data/spec-live/storage/scp_spec.rb +193 -0
- data/spec-live/syncer/cloud/s3_spec.rb +124 -0
- data/spec/archive_spec.rb +335 -0
- data/spec/cleaner_spec.rb +312 -0
- data/spec/cli/helpers_spec.rb +301 -0
- data/spec/cli/utility_spec.rb +411 -0
- data/spec/compressor/base_spec.rb +52 -0
- data/spec/compressor/bzip2_spec.rb +217 -0
- data/spec/compressor/custom_spec.rb +106 -0
- data/spec/compressor/gzip_spec.rb +217 -0
- data/spec/compressor/lzma_spec.rb +123 -0
- data/spec/compressor/pbzip2_spec.rb +165 -0
- data/spec/config_spec.rb +321 -0
- data/spec/configuration/helpers_spec.rb +247 -0
- data/spec/configuration/store_spec.rb +39 -0
- data/spec/configuration_spec.rb +62 -0
- data/spec/database/base_spec.rb +63 -0
- data/spec/database/mongodb_spec.rb +510 -0
- data/spec/database/mysql_spec.rb +411 -0
- data/spec/database/postgresql_spec.rb +353 -0
- data/spec/database/redis_spec.rb +334 -0
- data/spec/database/riak_spec.rb +176 -0
- data/spec/dependency_spec.rb +51 -0
- data/spec/encryptor/base_spec.rb +40 -0
- data/spec/encryptor/gpg_spec.rb +909 -0
- data/spec/encryptor/open_ssl_spec.rb +148 -0
- data/spec/errors_spec.rb +306 -0
- data/spec/hooks_spec.rb +35 -0
- data/spec/logger_spec.rb +367 -0
- data/spec/model_spec.rb +694 -0
- data/spec/notifier/base_spec.rb +104 -0
- data/spec/notifier/campfire_spec.rb +217 -0
- data/spec/notifier/hipchat_spec.rb +211 -0
- data/spec/notifier/mail_spec.rb +316 -0
- data/spec/notifier/prowl_spec.rb +138 -0
- data/spec/notifier/pushover_spec.rb +123 -0
- data/spec/notifier/twitter_spec.rb +153 -0
- data/spec/package_spec.rb +61 -0
- data/spec/packager_spec.rb +213 -0
- data/spec/pipeline_spec.rb +259 -0
- data/spec/spec_helper.rb +60 -0
- data/spec/splitter_spec.rb +120 -0
- data/spec/storage/base_spec.rb +166 -0
- data/spec/storage/cloudfiles_spec.rb +254 -0
- data/spec/storage/cycler_spec.rb +247 -0
- data/spec/storage/dropbox_spec.rb +480 -0
- data/spec/storage/ftp_spec.rb +271 -0
- data/spec/storage/local_spec.rb +259 -0
- data/spec/storage/ninefold_spec.rb +343 -0
- data/spec/storage/rsync_spec.rb +362 -0
- data/spec/storage/s3_spec.rb +245 -0
- data/spec/storage/scp_spec.rb +233 -0
- data/spec/storage/sftp_spec.rb +244 -0
- data/spec/syncer/base_spec.rb +109 -0
- data/spec/syncer/cloud/base_spec.rb +515 -0
- data/spec/syncer/cloud/cloud_files_spec.rb +181 -0
- data/spec/syncer/cloud/s3_spec.rb +174 -0
- data/spec/syncer/rsync/base_spec.rb +98 -0
- data/spec/syncer/rsync/local_spec.rb +149 -0
- data/spec/syncer/rsync/pull_spec.rb +98 -0
- data/spec/syncer/rsync/push_spec.rb +333 -0
- data/spec/version_spec.rb +21 -0
- data/templates/cli/utility/archive +25 -0
- data/templates/cli/utility/compressor/bzip2 +4 -0
- data/templates/cli/utility/compressor/custom +11 -0
- data/templates/cli/utility/compressor/gzip +4 -0
- data/templates/cli/utility/compressor/lzma +10 -0
- data/templates/cli/utility/compressor/pbzip2 +10 -0
- data/templates/cli/utility/config +32 -0
- data/templates/cli/utility/database/mongodb +18 -0
- data/templates/cli/utility/database/mysql +21 -0
- data/templates/cli/utility/database/postgresql +17 -0
- data/templates/cli/utility/database/redis +16 -0
- data/templates/cli/utility/database/riak +11 -0
- data/templates/cli/utility/encryptor/gpg +27 -0
- data/templates/cli/utility/encryptor/openssl +9 -0
- data/templates/cli/utility/model.erb +23 -0
- data/templates/cli/utility/notifier/campfire +12 -0
- data/templates/cli/utility/notifier/hipchat +15 -0
- data/templates/cli/utility/notifier/mail +22 -0
- data/templates/cli/utility/notifier/prowl +11 -0
- data/templates/cli/utility/notifier/pushover +11 -0
- data/templates/cli/utility/notifier/twitter +13 -0
- data/templates/cli/utility/splitter +7 -0
- data/templates/cli/utility/storage/cloud_files +22 -0
- data/templates/cli/utility/storage/dropbox +20 -0
- data/templates/cli/utility/storage/ftp +12 -0
- data/templates/cli/utility/storage/local +7 -0
- data/templates/cli/utility/storage/ninefold +9 -0
- data/templates/cli/utility/storage/rsync +11 -0
- data/templates/cli/utility/storage/s3 +19 -0
- data/templates/cli/utility/storage/scp +11 -0
- data/templates/cli/utility/storage/sftp +11 -0
- data/templates/cli/utility/syncer/cloud_files +46 -0
- data/templates/cli/utility/syncer/rsync_local +12 -0
- data/templates/cli/utility/syncer/rsync_pull +17 -0
- data/templates/cli/utility/syncer/rsync_push +17 -0
- data/templates/cli/utility/syncer/s3 +43 -0
- data/templates/general/links +11 -0
- data/templates/general/version.erb +2 -0
- data/templates/notifier/mail/failure.erb +9 -0
- data/templates/notifier/mail/success.erb +7 -0
- data/templates/notifier/mail/warning.erb +9 -0
- data/templates/storage/dropbox/authorization_url.erb +6 -0
- data/templates/storage/dropbox/authorized.erb +4 -0
- data/templates/storage/dropbox/cache_file_written.erb +10 -0
- metadata +277 -0
@@ -0,0 +1,75 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
module Backup
|
4
|
+
class Splitter
|
5
|
+
include Backup::CLI::Helpers
|
6
|
+
|
7
|
+
def initialize(model, chunk_size)
|
8
|
+
@model = model
|
9
|
+
@chunk_size = chunk_size
|
10
|
+
end
|
11
|
+
|
12
|
+
##
|
13
|
+
# This is called as part of the procedure used to build the final
|
14
|
+
# backup package file(s). It yields it's portion of the command line
|
15
|
+
# for this procedure, which will split the data being piped into it
|
16
|
+
# into multiple files, based on the @chunk_size.
|
17
|
+
# Once the packaging procedure is complete, it will return and
|
18
|
+
# @package.chunk_suffixes will be set based on the resulting files.
|
19
|
+
def split_with
|
20
|
+
before_packaging
|
21
|
+
yield @split_command
|
22
|
+
after_packaging
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
##
|
28
|
+
# The `split` command reads from $stdin and will store it's output in
|
29
|
+
# multiple files, based on the @chunk_size. The files will be
|
30
|
+
# written using the given `prefix`, which is the full path to the
|
31
|
+
# final @package.basename, plus a '-' separator. This `prefix` will then
|
32
|
+
# be suffixed using 'aa', 'ab', and so on... for each file.
|
33
|
+
def before_packaging
|
34
|
+
@package = @model.package
|
35
|
+
Logger.message "Splitter configured with a chunk size of " +
|
36
|
+
"#{ @chunk_size }MB."
|
37
|
+
|
38
|
+
@split_command = "#{ utility(:split) } -b #{ @chunk_size }m - " +
|
39
|
+
"'#{ File.join(Config.tmp_path, @package.basename + '-') }'"
|
40
|
+
end
|
41
|
+
|
42
|
+
##
|
43
|
+
# Finds the resulting files from the packaging procedure
|
44
|
+
# and stores an Array of suffixes used in @package.chunk_suffixes.
|
45
|
+
# If the @chunk_size was never reached and only one file
|
46
|
+
# was written, that file will be suffixed with '-aa'.
|
47
|
+
# In which case, it will simply remove the suffix from the filename.
|
48
|
+
def after_packaging
|
49
|
+
suffixes = chunk_suffixes
|
50
|
+
if suffixes == ['aa']
|
51
|
+
FileUtils.mv(
|
52
|
+
File.join(Config.tmp_path, @package.basename + '-aa'),
|
53
|
+
File.join(Config.tmp_path, @package.basename)
|
54
|
+
)
|
55
|
+
else
|
56
|
+
@package.chunk_suffixes = suffixes
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
##
|
61
|
+
# Returns an array of suffixes for each chunk, in alphabetical order.
|
62
|
+
# For example: [aa, ab, ac, ad, ae]
|
63
|
+
def chunk_suffixes
|
64
|
+
chunks.map {|chunk| File.extname(chunk).split('-').last }.sort
|
65
|
+
end
|
66
|
+
|
67
|
+
##
|
68
|
+
# Returns an array of full paths to the backup chunks.
|
69
|
+
# Chunks are sorted in alphabetical order.
|
70
|
+
def chunks
|
71
|
+
Dir[File.join(Config.tmp_path, @package.basename + '-*')].sort
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
module Backup
|
4
|
+
module Storage
|
5
|
+
class Base
|
6
|
+
include Backup::Configuration::Helpers
|
7
|
+
|
8
|
+
##
|
9
|
+
# Sets the limit to how many backups to keep in the remote location.
|
10
|
+
# If exceeded, the oldest will be removed to make room for the newest
|
11
|
+
attr_accessor :keep
|
12
|
+
|
13
|
+
##
|
14
|
+
# (Optional)
|
15
|
+
# User-defined string used to uniquely identify multiple storages of the
|
16
|
+
# same type. This will be appended to the YAML storage file used for
|
17
|
+
# cycling backups.
|
18
|
+
attr_accessor :storage_id
|
19
|
+
|
20
|
+
##
|
21
|
+
# Creates a new instance of the storage object
|
22
|
+
# * Called with super(model, storage_id) from each subclass
|
23
|
+
def initialize(model, storage_id = nil)
|
24
|
+
load_defaults!
|
25
|
+
@model = model
|
26
|
+
@storage_id = storage_id
|
27
|
+
end
|
28
|
+
|
29
|
+
##
|
30
|
+
# Performs the backup transfer
|
31
|
+
def perform!
|
32
|
+
@package = @model.package
|
33
|
+
transfer!
|
34
|
+
cycle!
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
##
|
40
|
+
# Provider defaults to false. Overridden when using a service-based
|
41
|
+
# storage such as Amazon S3, Rackspace Cloud Files or Dropbox
|
42
|
+
def provider
|
43
|
+
false
|
44
|
+
end
|
45
|
+
|
46
|
+
##
|
47
|
+
# Each subclass must define a +path+ where remote files will be stored
|
48
|
+
def path; end
|
49
|
+
|
50
|
+
##
|
51
|
+
# Return the storage name, with optional storage_id
|
52
|
+
def storage_name
|
53
|
+
self.class.to_s.sub('Backup::', '') +
|
54
|
+
(storage_id ? " (#{storage_id})" : '')
|
55
|
+
end
|
56
|
+
|
57
|
+
##
|
58
|
+
# Returns the local path
|
59
|
+
# This is where any Package to be transferred is located.
|
60
|
+
def local_path
|
61
|
+
Config.tmp_path
|
62
|
+
end
|
63
|
+
|
64
|
+
##
|
65
|
+
# Returns the remote path for the given Package
|
66
|
+
# This is where the Package will be stored, or was previously stored.
|
67
|
+
def remote_path_for(package)
|
68
|
+
File.join(path, package.trigger, package.time)
|
69
|
+
end
|
70
|
+
|
71
|
+
##
|
72
|
+
# Yields two arguments to the given block: "local_file, remote_file"
|
73
|
+
# The local_file is the full file name:
|
74
|
+
# e.g. "2011.08.30.11.00.02.backup.tar.enc"
|
75
|
+
# The remote_file is the full file name, minus the timestamp:
|
76
|
+
# e.g. "backup.tar.enc"
|
77
|
+
def files_to_transfer_for(package)
|
78
|
+
package.filenames.each do |filename|
|
79
|
+
yield filename, filename[20..-1]
|
80
|
+
end
|
81
|
+
end
|
82
|
+
alias :transferred_files_for :files_to_transfer_for
|
83
|
+
|
84
|
+
##
|
85
|
+
# Adds the current package being stored to the YAML cycle data file
|
86
|
+
# and will remove any old Package file(s) when the storage limit
|
87
|
+
# set by #keep is exceeded. Any errors raised while attempting to
|
88
|
+
# remove older packages will be rescued and a warning will be logged
|
89
|
+
# containing the original error message.
|
90
|
+
def cycle!
|
91
|
+
return unless keep.to_i > 0
|
92
|
+
Logger.message "#{ storage_name }: Cycling Started..."
|
93
|
+
Cycler.cycle!(self, @package)
|
94
|
+
Logger.message "#{ storage_name }: Cycling Complete!"
|
95
|
+
end
|
96
|
+
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
##
|
4
|
+
# Only load the Fog gem when the Backup::Storage::CloudFiles class is loaded
|
5
|
+
Backup::Dependency.load('fog')
|
6
|
+
|
7
|
+
module Backup
|
8
|
+
module Storage
|
9
|
+
class CloudFiles < Base
|
10
|
+
|
11
|
+
##
|
12
|
+
# Rackspace Cloud Files Credentials
|
13
|
+
attr_accessor :username, :api_key, :auth_url
|
14
|
+
|
15
|
+
##
|
16
|
+
# Rackspace Service Net
|
17
|
+
# (LAN-based transfers to avoid charges and improve performance)
|
18
|
+
attr_accessor :servicenet
|
19
|
+
|
20
|
+
##
|
21
|
+
# Rackspace Cloud Files container name and path
|
22
|
+
attr_accessor :container, :path
|
23
|
+
|
24
|
+
##
|
25
|
+
# Creates a new instance of the storage object
|
26
|
+
def initialize(model, storage_id = nil, &block)
|
27
|
+
super(model, storage_id)
|
28
|
+
|
29
|
+
@servicenet ||= false
|
30
|
+
@path ||= 'backups'
|
31
|
+
|
32
|
+
instance_eval(&block) if block_given?
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
##
|
38
|
+
# This is the provider that Fog uses for the Cloud Files Storage
|
39
|
+
def provider
|
40
|
+
'Rackspace'
|
41
|
+
end
|
42
|
+
|
43
|
+
##
|
44
|
+
# Establishes a connection to Rackspace Cloud Files
|
45
|
+
def connection
|
46
|
+
@connection ||= Fog::Storage.new(
|
47
|
+
:provider => provider,
|
48
|
+
:rackspace_username => username,
|
49
|
+
:rackspace_api_key => api_key,
|
50
|
+
:rackspace_auth_url => auth_url,
|
51
|
+
:rackspace_servicenet => servicenet
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
##
|
56
|
+
# Transfers the archived file to the specified Cloud Files container
|
57
|
+
def transfer!
|
58
|
+
remote_path = remote_path_for(@package)
|
59
|
+
|
60
|
+
files_to_transfer_for(@package) do |local_file, remote_file|
|
61
|
+
Logger.message "#{storage_name} started transferring '#{ local_file }'."
|
62
|
+
|
63
|
+
File.open(File.join(local_path, local_file), 'r') do |file|
|
64
|
+
connection.put_object(
|
65
|
+
container, File.join(remote_path, remote_file), file
|
66
|
+
)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
##
|
72
|
+
# Removes the transferred archive file(s) from the storage location.
|
73
|
+
# Any error raised will be rescued during Cycling
|
74
|
+
# and a warning will be logged, containing the error message.
|
75
|
+
def remove!(package)
|
76
|
+
remote_path = remote_path_for(package)
|
77
|
+
|
78
|
+
transferred_files_for(package) do |local_file, remote_file|
|
79
|
+
Logger.message "#{storage_name} started removing '#{ local_file }' " +
|
80
|
+
"from container '#{ container }'."
|
81
|
+
connection.delete_object(container, File.join(remote_path, remote_file))
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,117 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
module Backup
|
4
|
+
module Storage
|
5
|
+
module Cycler
|
6
|
+
class << self
|
7
|
+
|
8
|
+
##
|
9
|
+
# Adds the given +package+ to the YAML storage file corresponding
|
10
|
+
# to the given +storage+ and Package#trigger (Model#trigger).
|
11
|
+
# Then, calls the +storage+ to remove the files for any older
|
12
|
+
# packages that were removed from the YAML storage file.
|
13
|
+
def cycle!(storage, package)
|
14
|
+
@storage, @package = storage, package
|
15
|
+
@storage_file = storage_file
|
16
|
+
|
17
|
+
update_storage_file!
|
18
|
+
remove_packages!
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
##
|
24
|
+
# Updates the YAML data file according to the #keep setting
|
25
|
+
# for the storage and sets the @packages_to_remove
|
26
|
+
def update_storage_file!
|
27
|
+
packages = yaml_load.unshift(@package)
|
28
|
+
excess = packages.count - @storage.keep.to_i
|
29
|
+
@packages_to_remove = (excess > 0) ? packages.pop(excess) : []
|
30
|
+
yaml_save(packages)
|
31
|
+
end
|
32
|
+
|
33
|
+
##
|
34
|
+
# Calls the @storage to remove any old packages
|
35
|
+
# which were cycled out of the storage file.
|
36
|
+
def remove_packages!
|
37
|
+
@packages_to_remove.each do |pkg|
|
38
|
+
begin
|
39
|
+
@storage.send(:remove!, pkg)
|
40
|
+
rescue => err
|
41
|
+
Logger.warn Errors::Storage::CyclerError.wrap(err, <<-EOS)
|
42
|
+
There was a problem removing the following package:
|
43
|
+
Trigger: #{pkg.trigger} :: Dated: #{pkg.time}
|
44
|
+
Package included the following #{ pkg.filenames.count } file(s):
|
45
|
+
#{ pkg.filenames.join("\n") }
|
46
|
+
EOS
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
##
|
52
|
+
# Return full path to the YAML data file,
|
53
|
+
# based on the current values of @storage and @package
|
54
|
+
def storage_file
|
55
|
+
type = @storage.class.to_s.split('::').last
|
56
|
+
suffix = @storage.storage_id.to_s.strip.gsub(/[\W\s]/, '_')
|
57
|
+
filename = suffix.empty? ? type : "#{type}-#{suffix}"
|
58
|
+
File.join(Config.data_path, @package.trigger, "#{filename}.yml")
|
59
|
+
end
|
60
|
+
|
61
|
+
##
|
62
|
+
# Load Package objects from YAML file.
|
63
|
+
# Returns an Array, sorted by @time descending.
|
64
|
+
# i.e. most recent is objects[0]
|
65
|
+
def yaml_load
|
66
|
+
packages = []
|
67
|
+
if File.exist?(@storage_file) && !File.zero?(@storage_file)
|
68
|
+
packages = check_upgrade(
|
69
|
+
YAML.load_file(@storage_file).sort do |a, b|
|
70
|
+
b.instance_variable_get(:@time) <=> a.instance_variable_get(:@time)
|
71
|
+
end
|
72
|
+
)
|
73
|
+
end
|
74
|
+
packages
|
75
|
+
end
|
76
|
+
|
77
|
+
##
|
78
|
+
# Store the given package objects to the YAML data file.
|
79
|
+
def yaml_save(packages)
|
80
|
+
File.open(@storage_file, 'w') do |file|
|
81
|
+
file.write(packages.to_yaml)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
##
|
86
|
+
# Upgrade the objects loaded from the YAML file, if needed.
|
87
|
+
def check_upgrade(objects)
|
88
|
+
if objects.any? {|obj| obj.class.to_s =~ /Backup::Storage/ }
|
89
|
+
# Version <= 3.0.20
|
90
|
+
model = @storage.instance_variable_get(:@model)
|
91
|
+
v3_0_20 = objects.any? {|obj| obj.instance_variable_defined?(:@version) }
|
92
|
+
objects.map! do |obj|
|
93
|
+
if v3_0_20 # Version == 3.0.20
|
94
|
+
filename = obj.instance_variable_get(:@filename)[20..-1]
|
95
|
+
chunk_suffixes = obj.instance_variable_get(:@chunk_suffixes)
|
96
|
+
else # Version <= 3.0.19
|
97
|
+
filename = obj.instance_variable_get(:@remote_file)[20..-1]
|
98
|
+
chunk_suffixes = []
|
99
|
+
end
|
100
|
+
time = obj.instance_variable_get(:@time)
|
101
|
+
extension = filename.match(/\.(tar.*)$/)[1]
|
102
|
+
|
103
|
+
package = Backup::Package.new(model)
|
104
|
+
package.instance_variable_set(:@time, time)
|
105
|
+
package.extension = extension
|
106
|
+
package.chunk_suffixes = chunk_suffixes
|
107
|
+
|
108
|
+
package
|
109
|
+
end
|
110
|
+
end
|
111
|
+
objects
|
112
|
+
end
|
113
|
+
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
@@ -0,0 +1,178 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
##
|
4
|
+
# Only load the Dropbox gem when the Backup::Storage::Dropbox class is loaded
|
5
|
+
Backup::Dependency.load('dropbox-sdk')
|
6
|
+
|
7
|
+
module Backup
|
8
|
+
module Storage
|
9
|
+
class Dropbox < Base
|
10
|
+
|
11
|
+
##
|
12
|
+
# Dropbox API credentials
|
13
|
+
attr_accessor :api_key, :api_secret
|
14
|
+
|
15
|
+
##
|
16
|
+
# Dropbox Access Type
|
17
|
+
# Valid values are:
|
18
|
+
# :app_folder (default)
|
19
|
+
# :dropbox (full access)
|
20
|
+
attr_accessor :access_type
|
21
|
+
|
22
|
+
##
|
23
|
+
# Path to where the backups will be stored
|
24
|
+
attr_accessor :path
|
25
|
+
|
26
|
+
attr_deprecate :email, :version => '3.0.17'
|
27
|
+
attr_deprecate :password, :version => '3.0.17'
|
28
|
+
|
29
|
+
attr_deprecate :timeout, :version => '3.0.21'
|
30
|
+
|
31
|
+
##
|
32
|
+
# Creates a new instance of the storage object
|
33
|
+
def initialize(model, storage_id = nil, &block)
|
34
|
+
super(model, storage_id)
|
35
|
+
|
36
|
+
@path ||= 'backups'
|
37
|
+
@access_type ||= :app_folder
|
38
|
+
|
39
|
+
instance_eval(&block) if block_given?
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
##
|
45
|
+
# The initial connection to Dropbox will provide the user with an
|
46
|
+
# authorization url. The user must open this URL and confirm that the
|
47
|
+
# authorization successfully took place. If this is the case, then the
|
48
|
+
# user hits 'enter' and the session will be properly established.
|
49
|
+
# Immediately after establishing the session, the session will be
|
50
|
+
# serialized and written to a cache file in Backup::Config.cache_path.
|
51
|
+
# The cached file will be used from that point on to re-establish a
|
52
|
+
# connection with Dropbox at a later time. This allows the user to avoid
|
53
|
+
# having to go to a new Dropbox URL to authorize over and over again.
|
54
|
+
def connection
|
55
|
+
return @connection if @connection
|
56
|
+
|
57
|
+
unless session = cached_session
|
58
|
+
Logger.message "Creating a new session!"
|
59
|
+
session = create_write_and_return_new_session!
|
60
|
+
end
|
61
|
+
|
62
|
+
# will raise an error if session not authorized
|
63
|
+
@connection = DropboxClient.new(session, access_type)
|
64
|
+
|
65
|
+
rescue => err
|
66
|
+
raise Errors::Storage::Dropbox::ConnectionError.wrap(err)
|
67
|
+
end
|
68
|
+
|
69
|
+
##
|
70
|
+
# Attempt to load a cached session
|
71
|
+
def cached_session
|
72
|
+
session = false
|
73
|
+
if cache_exists?
|
74
|
+
begin
|
75
|
+
session = DropboxSession.deserialize(File.read(cached_file))
|
76
|
+
Logger.message "Session data loaded from cache!"
|
77
|
+
|
78
|
+
rescue => err
|
79
|
+
Logger.warn Errors::Storage::Dropbox::CacheError.wrap(err, <<-EOS)
|
80
|
+
Could not read session data from cache.
|
81
|
+
Cache data might be corrupt.
|
82
|
+
EOS
|
83
|
+
end
|
84
|
+
end
|
85
|
+
session
|
86
|
+
end
|
87
|
+
|
88
|
+
##
|
89
|
+
# Transfers the archived file to the specified Dropbox folder
|
90
|
+
def transfer!
|
91
|
+
remote_path = remote_path_for(@package)
|
92
|
+
|
93
|
+
files_to_transfer_for(@package) do |local_file, remote_file|
|
94
|
+
Logger.message "#{storage_name} started transferring '#{ local_file }'."
|
95
|
+
File.open(File.join(local_path, local_file), 'r') do |file|
|
96
|
+
connection.put_file(File.join(remote_path, remote_file), file)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
##
|
102
|
+
# Removes the transferred archive file(s) from the storage location.
|
103
|
+
# Any error raised will be rescued during Cycling
|
104
|
+
# and a warning will be logged, containing the error message.
|
105
|
+
def remove!(package)
|
106
|
+
remote_path = remote_path_for(package)
|
107
|
+
|
108
|
+
messages = []
|
109
|
+
transferred_files_for(package) do |local_file, remote_file|
|
110
|
+
messages << "#{storage_name} started removing " +
|
111
|
+
"'#{ local_file }' from Dropbox."
|
112
|
+
end
|
113
|
+
Logger.message messages.join("\n")
|
114
|
+
|
115
|
+
connection.file_delete(remote_path)
|
116
|
+
end
|
117
|
+
|
118
|
+
##
|
119
|
+
# Returns the path to the cached file
|
120
|
+
def cached_file
|
121
|
+
File.join(Config.cache_path, api_key + api_secret)
|
122
|
+
end
|
123
|
+
|
124
|
+
##
|
125
|
+
# Checks to see if the cache file exists
|
126
|
+
def cache_exists?
|
127
|
+
File.exist?(cached_file)
|
128
|
+
end
|
129
|
+
|
130
|
+
##
|
131
|
+
# Serializes and writes the Dropbox session to a cache file
|
132
|
+
def write_cache!(session)
|
133
|
+
File.open(cached_file, "w") do |cache_file|
|
134
|
+
cache_file.write(session.serialize)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
##
|
139
|
+
# Create a new session, write a serialized version of it to the
|
140
|
+
# .cache directory, and return the session object
|
141
|
+
def create_write_and_return_new_session!
|
142
|
+
require 'timeout'
|
143
|
+
|
144
|
+
session = DropboxSession.new(api_key, api_secret)
|
145
|
+
|
146
|
+
# grab the request token for session
|
147
|
+
session.get_request_token
|
148
|
+
|
149
|
+
template = Backup::Template.new(
|
150
|
+
{:session => session, :cached_file => cached_file}
|
151
|
+
)
|
152
|
+
template.render("storage/dropbox/authorization_url.erb")
|
153
|
+
|
154
|
+
# wait for user to hit 'return' to continue
|
155
|
+
Timeout::timeout(180) { STDIN.gets }
|
156
|
+
|
157
|
+
# this will raise an error if the user did not
|
158
|
+
# visit the authorization_url and grant access
|
159
|
+
#
|
160
|
+
# get the access token from the server
|
161
|
+
# this will be stored with the session in the cache file
|
162
|
+
session.get_access_token
|
163
|
+
|
164
|
+
template.render("storage/dropbox/authorized.erb")
|
165
|
+
write_cache!(session)
|
166
|
+
template.render("storage/dropbox/cache_file_written.erb")
|
167
|
+
|
168
|
+
session
|
169
|
+
|
170
|
+
rescue => err
|
171
|
+
raise Errors::Storage::Dropbox::AuthenticationError.wrap(
|
172
|
+
err, 'Could not create or authenticate a new session'
|
173
|
+
)
|
174
|
+
end
|
175
|
+
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|