backup-agoddard 3.0.27
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of backup-agoddard might be problematic. Click here for more details.
- data/.gitignore +8 -0
- data/.travis.yml +10 -0
- data/Gemfile +28 -0
- data/Guardfile +23 -0
- data/LICENSE.md +24 -0
- data/README.md +478 -0
- data/backup.gemspec +32 -0
- data/bin/backup +11 -0
- data/lib/backup.rb +133 -0
- data/lib/backup/archive.rb +117 -0
- data/lib/backup/binder.rb +22 -0
- data/lib/backup/cleaner.rb +121 -0
- data/lib/backup/cli/helpers.rb +93 -0
- data/lib/backup/cli/utility.rb +255 -0
- data/lib/backup/compressor/base.rb +35 -0
- data/lib/backup/compressor/bzip2.rb +50 -0
- data/lib/backup/compressor/custom.rb +53 -0
- data/lib/backup/compressor/gzip.rb +50 -0
- data/lib/backup/compressor/lzma.rb +52 -0
- data/lib/backup/compressor/pbzip2.rb +59 -0
- data/lib/backup/config.rb +174 -0
- data/lib/backup/configuration.rb +33 -0
- data/lib/backup/configuration/helpers.rb +130 -0
- data/lib/backup/configuration/store.rb +24 -0
- data/lib/backup/database/base.rb +53 -0
- data/lib/backup/database/mongodb.rb +230 -0
- data/lib/backup/database/mysql.rb +160 -0
- data/lib/backup/database/postgresql.rb +144 -0
- data/lib/backup/database/redis.rb +136 -0
- data/lib/backup/database/riak.rb +67 -0
- data/lib/backup/dependency.rb +108 -0
- data/lib/backup/encryptor/base.rb +29 -0
- data/lib/backup/encryptor/gpg.rb +760 -0
- data/lib/backup/encryptor/open_ssl.rb +72 -0
- data/lib/backup/errors.rb +124 -0
- data/lib/backup/hooks.rb +68 -0
- data/lib/backup/logger.rb +152 -0
- data/lib/backup/model.rb +409 -0
- data/lib/backup/notifier/base.rb +81 -0
- data/lib/backup/notifier/campfire.rb +155 -0
- data/lib/backup/notifier/hipchat.rb +93 -0
- data/lib/backup/notifier/mail.rb +206 -0
- data/lib/backup/notifier/prowl.rb +65 -0
- data/lib/backup/notifier/pushover.rb +88 -0
- data/lib/backup/notifier/twitter.rb +70 -0
- data/lib/backup/package.rb +47 -0
- data/lib/backup/packager.rb +100 -0
- data/lib/backup/pipeline.rb +110 -0
- data/lib/backup/splitter.rb +75 -0
- data/lib/backup/storage/base.rb +99 -0
- data/lib/backup/storage/cloudfiles.rb +87 -0
- data/lib/backup/storage/cycler.rb +117 -0
- data/lib/backup/storage/dropbox.rb +178 -0
- data/lib/backup/storage/ftp.rb +119 -0
- data/lib/backup/storage/local.rb +82 -0
- data/lib/backup/storage/ninefold.rb +116 -0
- data/lib/backup/storage/rsync.rb +149 -0
- data/lib/backup/storage/s3.rb +94 -0
- data/lib/backup/storage/scp.rb +99 -0
- data/lib/backup/storage/sftp.rb +108 -0
- data/lib/backup/syncer/base.rb +46 -0
- data/lib/backup/syncer/cloud/base.rb +247 -0
- data/lib/backup/syncer/cloud/cloud_files.rb +78 -0
- data/lib/backup/syncer/cloud/s3.rb +68 -0
- data/lib/backup/syncer/rsync/base.rb +49 -0
- data/lib/backup/syncer/rsync/local.rb +55 -0
- data/lib/backup/syncer/rsync/pull.rb +36 -0
- data/lib/backup/syncer/rsync/push.rb +116 -0
- data/lib/backup/template.rb +46 -0
- data/lib/backup/version.rb +43 -0
- data/spec-live/.gitignore +6 -0
- data/spec-live/README +7 -0
- data/spec-live/backups/config.rb +83 -0
- data/spec-live/backups/config.yml.template +46 -0
- data/spec-live/backups/models.rb +184 -0
- data/spec-live/compressor/custom_spec.rb +30 -0
- data/spec-live/compressor/gzip_spec.rb +30 -0
- data/spec-live/encryptor/gpg_keys.rb +239 -0
- data/spec-live/encryptor/gpg_spec.rb +287 -0
- data/spec-live/notifier/mail_spec.rb +121 -0
- data/spec-live/spec_helper.rb +151 -0
- data/spec-live/storage/dropbox_spec.rb +151 -0
- data/spec-live/storage/local_spec.rb +83 -0
- data/spec-live/storage/scp_spec.rb +193 -0
- data/spec-live/syncer/cloud/s3_spec.rb +124 -0
- data/spec/archive_spec.rb +335 -0
- data/spec/cleaner_spec.rb +312 -0
- data/spec/cli/helpers_spec.rb +301 -0
- data/spec/cli/utility_spec.rb +411 -0
- data/spec/compressor/base_spec.rb +52 -0
- data/spec/compressor/bzip2_spec.rb +217 -0
- data/spec/compressor/custom_spec.rb +106 -0
- data/spec/compressor/gzip_spec.rb +217 -0
- data/spec/compressor/lzma_spec.rb +123 -0
- data/spec/compressor/pbzip2_spec.rb +165 -0
- data/spec/config_spec.rb +321 -0
- data/spec/configuration/helpers_spec.rb +247 -0
- data/spec/configuration/store_spec.rb +39 -0
- data/spec/configuration_spec.rb +62 -0
- data/spec/database/base_spec.rb +63 -0
- data/spec/database/mongodb_spec.rb +510 -0
- data/spec/database/mysql_spec.rb +411 -0
- data/spec/database/postgresql_spec.rb +353 -0
- data/spec/database/redis_spec.rb +334 -0
- data/spec/database/riak_spec.rb +176 -0
- data/spec/dependency_spec.rb +51 -0
- data/spec/encryptor/base_spec.rb +40 -0
- data/spec/encryptor/gpg_spec.rb +909 -0
- data/spec/encryptor/open_ssl_spec.rb +148 -0
- data/spec/errors_spec.rb +306 -0
- data/spec/hooks_spec.rb +35 -0
- data/spec/logger_spec.rb +367 -0
- data/spec/model_spec.rb +694 -0
- data/spec/notifier/base_spec.rb +104 -0
- data/spec/notifier/campfire_spec.rb +217 -0
- data/spec/notifier/hipchat_spec.rb +211 -0
- data/spec/notifier/mail_spec.rb +316 -0
- data/spec/notifier/prowl_spec.rb +138 -0
- data/spec/notifier/pushover_spec.rb +123 -0
- data/spec/notifier/twitter_spec.rb +153 -0
- data/spec/package_spec.rb +61 -0
- data/spec/packager_spec.rb +213 -0
- data/spec/pipeline_spec.rb +259 -0
- data/spec/spec_helper.rb +60 -0
- data/spec/splitter_spec.rb +120 -0
- data/spec/storage/base_spec.rb +166 -0
- data/spec/storage/cloudfiles_spec.rb +254 -0
- data/spec/storage/cycler_spec.rb +247 -0
- data/spec/storage/dropbox_spec.rb +480 -0
- data/spec/storage/ftp_spec.rb +271 -0
- data/spec/storage/local_spec.rb +259 -0
- data/spec/storage/ninefold_spec.rb +343 -0
- data/spec/storage/rsync_spec.rb +362 -0
- data/spec/storage/s3_spec.rb +245 -0
- data/spec/storage/scp_spec.rb +233 -0
- data/spec/storage/sftp_spec.rb +244 -0
- data/spec/syncer/base_spec.rb +109 -0
- data/spec/syncer/cloud/base_spec.rb +515 -0
- data/spec/syncer/cloud/cloud_files_spec.rb +181 -0
- data/spec/syncer/cloud/s3_spec.rb +174 -0
- data/spec/syncer/rsync/base_spec.rb +98 -0
- data/spec/syncer/rsync/local_spec.rb +149 -0
- data/spec/syncer/rsync/pull_spec.rb +98 -0
- data/spec/syncer/rsync/push_spec.rb +333 -0
- data/spec/version_spec.rb +21 -0
- data/templates/cli/utility/archive +25 -0
- data/templates/cli/utility/compressor/bzip2 +4 -0
- data/templates/cli/utility/compressor/custom +11 -0
- data/templates/cli/utility/compressor/gzip +4 -0
- data/templates/cli/utility/compressor/lzma +10 -0
- data/templates/cli/utility/compressor/pbzip2 +10 -0
- data/templates/cli/utility/config +32 -0
- data/templates/cli/utility/database/mongodb +18 -0
- data/templates/cli/utility/database/mysql +21 -0
- data/templates/cli/utility/database/postgresql +17 -0
- data/templates/cli/utility/database/redis +16 -0
- data/templates/cli/utility/database/riak +11 -0
- data/templates/cli/utility/encryptor/gpg +27 -0
- data/templates/cli/utility/encryptor/openssl +9 -0
- data/templates/cli/utility/model.erb +23 -0
- data/templates/cli/utility/notifier/campfire +12 -0
- data/templates/cli/utility/notifier/hipchat +15 -0
- data/templates/cli/utility/notifier/mail +22 -0
- data/templates/cli/utility/notifier/prowl +11 -0
- data/templates/cli/utility/notifier/pushover +11 -0
- data/templates/cli/utility/notifier/twitter +13 -0
- data/templates/cli/utility/splitter +7 -0
- data/templates/cli/utility/storage/cloud_files +22 -0
- data/templates/cli/utility/storage/dropbox +20 -0
- data/templates/cli/utility/storage/ftp +12 -0
- data/templates/cli/utility/storage/local +7 -0
- data/templates/cli/utility/storage/ninefold +9 -0
- data/templates/cli/utility/storage/rsync +11 -0
- data/templates/cli/utility/storage/s3 +19 -0
- data/templates/cli/utility/storage/scp +11 -0
- data/templates/cli/utility/storage/sftp +11 -0
- data/templates/cli/utility/syncer/cloud_files +46 -0
- data/templates/cli/utility/syncer/rsync_local +12 -0
- data/templates/cli/utility/syncer/rsync_pull +17 -0
- data/templates/cli/utility/syncer/rsync_push +17 -0
- data/templates/cli/utility/syncer/s3 +43 -0
- data/templates/general/links +11 -0
- data/templates/general/version.erb +2 -0
- data/templates/notifier/mail/failure.erb +9 -0
- data/templates/notifier/mail/success.erb +7 -0
- data/templates/notifier/mail/warning.erb +9 -0
- data/templates/storage/dropbox/authorization_url.erb +6 -0
- data/templates/storage/dropbox/authorized.erb +4 -0
- data/templates/storage/dropbox/cache_file_written.erb +10 -0
- metadata +277 -0
@@ -0,0 +1,65 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
##
|
4
|
+
# Only load the Prowler gem when using Prowler notifications
|
5
|
+
Backup::Dependency.load('prowler')
|
6
|
+
|
7
|
+
module Backup
|
8
|
+
module Notifier
|
9
|
+
class Prowl < Base
|
10
|
+
|
11
|
+
##
|
12
|
+
# Application name
|
13
|
+
# Tell something like your server name. Example: "Server1 Backup"
|
14
|
+
attr_accessor :application
|
15
|
+
|
16
|
+
##
|
17
|
+
# API-Key
|
18
|
+
# Create a Prowl account and request an API key on prowlapp.com.
|
19
|
+
attr_accessor :api_key
|
20
|
+
|
21
|
+
def initialize(model, &block)
|
22
|
+
super(model)
|
23
|
+
|
24
|
+
instance_eval(&block) if block_given?
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
##
|
30
|
+
# Notify the user of the backup operation results.
|
31
|
+
# `status` indicates one of the following:
|
32
|
+
#
|
33
|
+
# `:success`
|
34
|
+
# : The backup completed successfully.
|
35
|
+
# : Notification will be sent if `on_success` was set to `true`
|
36
|
+
#
|
37
|
+
# `:warning`
|
38
|
+
# : The backup completed successfully, but warnings were logged
|
39
|
+
# : Notification will be sent, including a copy of the current
|
40
|
+
# : backup log, if `on_warning` was set to `true`
|
41
|
+
#
|
42
|
+
# `:failure`
|
43
|
+
# : The backup operation failed.
|
44
|
+
# : Notification will be sent, including the Exception which caused
|
45
|
+
# : the failure, the Exception's backtrace, a copy of the current
|
46
|
+
# : backup log and other information if `on_failure` was set to `true`
|
47
|
+
#
|
48
|
+
def notify!(status)
|
49
|
+
name = case status
|
50
|
+
when :success then 'Success'
|
51
|
+
when :warning then 'Warning'
|
52
|
+
when :failure then 'Failure'
|
53
|
+
end
|
54
|
+
message = '[Backup::%s]' % name
|
55
|
+
send_message(message)
|
56
|
+
end
|
57
|
+
|
58
|
+
def send_message(message)
|
59
|
+
client = Prowler.new(:application => application, :api_key => api_key)
|
60
|
+
client.notify(message, "#{@model.label} (#{@model.trigger})")
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'net/https'
|
3
|
+
|
4
|
+
module Backup
|
5
|
+
module Notifier
|
6
|
+
class Pushover < Base
|
7
|
+
|
8
|
+
##
|
9
|
+
# The API User Token
|
10
|
+
attr_accessor :user
|
11
|
+
|
12
|
+
##
|
13
|
+
# The API Application Token
|
14
|
+
attr_accessor :token
|
15
|
+
|
16
|
+
##
|
17
|
+
# The user's device identifier to sent he message directly to that device rather than all of the user's devices
|
18
|
+
attr_accessor :device
|
19
|
+
|
20
|
+
##
|
21
|
+
# The message title
|
22
|
+
attr_accessor :title
|
23
|
+
|
24
|
+
##
|
25
|
+
# The priority of the notification
|
26
|
+
attr_accessor :priority
|
27
|
+
|
28
|
+
def initialize(model, &block)
|
29
|
+
super(model)
|
30
|
+
|
31
|
+
instance_eval(&block) if block_given?
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
|
36
|
+
##
|
37
|
+
# Notify the user of the backup operation results.
|
38
|
+
# `status` indicates one of the following:
|
39
|
+
#
|
40
|
+
# `:success`
|
41
|
+
# : The backup completed successfully.
|
42
|
+
# : Notification will be sent if `on_success` was set to `true`
|
43
|
+
#
|
44
|
+
# `:warning`
|
45
|
+
# : The backup completed successfully, but warnings were logged
|
46
|
+
# : Notification will be sent, including a copy of the current
|
47
|
+
# : backup log, if `on_warning` was set to `true`
|
48
|
+
#
|
49
|
+
# `:failure`
|
50
|
+
# : The backup operation failed.
|
51
|
+
# : Notification will be sent, including the Exception which caused
|
52
|
+
# : the failure, the Exception's backtrace, a copy of the current
|
53
|
+
# : backup log and other information if `on_failure` was set to `true`
|
54
|
+
#
|
55
|
+
def notify!(status)
|
56
|
+
name = case status
|
57
|
+
when :success then 'Success'
|
58
|
+
when :failure then 'Failure'
|
59
|
+
when :warning then 'Warning'
|
60
|
+
end
|
61
|
+
message = "[Backup::%s] #{@model.label} (#{@model.trigger})" % name
|
62
|
+
|
63
|
+
send_message(message)
|
64
|
+
end
|
65
|
+
|
66
|
+
# Push a message via the Pushover API
|
67
|
+
def send_message(message)
|
68
|
+
url = URI.parse("https://api.pushover.net/1/messages.json")
|
69
|
+
|
70
|
+
request = Net::HTTP::Post.new(url.path)
|
71
|
+
request.set_form_data(parameters.merge ({:message => message}))
|
72
|
+
response = Net::HTTP.new(url.host, url.port)
|
73
|
+
|
74
|
+
response.use_ssl = true
|
75
|
+
response.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
76
|
+
|
77
|
+
response.start {|http| http.request(request) }
|
78
|
+
end
|
79
|
+
|
80
|
+
# List available parameters
|
81
|
+
def parameters
|
82
|
+
@values = {}
|
83
|
+
[:token, :user, :message, :title, :priority, :device].each { |k| @values.merge! k => self.instance_variable_get("@#{k}") }
|
84
|
+
@values
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
##
|
4
|
+
# Only load the Twitter gem when using Twitter notifications
|
5
|
+
Backup::Dependency.load('twitter')
|
6
|
+
|
7
|
+
module Backup
|
8
|
+
module Notifier
|
9
|
+
class Twitter < Base
|
10
|
+
|
11
|
+
##
|
12
|
+
# Twitter consumer key credentials
|
13
|
+
attr_accessor :consumer_key, :consumer_secret
|
14
|
+
|
15
|
+
##
|
16
|
+
# OAuth credentials
|
17
|
+
attr_accessor :oauth_token, :oauth_token_secret
|
18
|
+
|
19
|
+
def initialize(model, &block)
|
20
|
+
super(model)
|
21
|
+
|
22
|
+
instance_eval(&block) if block_given?
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
##
|
28
|
+
# Notify the user of the backup operation results.
|
29
|
+
# `status` indicates one of the following:
|
30
|
+
#
|
31
|
+
# `:success`
|
32
|
+
# : The backup completed successfully.
|
33
|
+
# : Notification will be sent if `on_success` was set to `true`
|
34
|
+
#
|
35
|
+
# `:warning`
|
36
|
+
# : The backup completed successfully, but warnings were logged
|
37
|
+
# : Notification will be sent, including a copy of the current
|
38
|
+
# : backup log, if `on_warning` was set to `true`
|
39
|
+
#
|
40
|
+
# `:failure`
|
41
|
+
# : The backup operation failed.
|
42
|
+
# : Notification will be sent, including the Exception which caused
|
43
|
+
# : the failure, the Exception's backtrace, a copy of the current
|
44
|
+
# : backup log and other information if `on_failure` was set to `true`
|
45
|
+
#
|
46
|
+
def notify!(status)
|
47
|
+
name = case status
|
48
|
+
when :success then 'Success'
|
49
|
+
when :warning then 'Warning'
|
50
|
+
when :failure then 'Failure'
|
51
|
+
end
|
52
|
+
message = "[Backup::%s] #{@model.label} (#{@model.trigger}) (@ #{@model.time})" % name
|
53
|
+
send_message(message)
|
54
|
+
end
|
55
|
+
|
56
|
+
def send_message(message)
|
57
|
+
::Twitter.configure do |config|
|
58
|
+
config.consumer_key = @consumer_key
|
59
|
+
config.consumer_secret = @consumer_secret
|
60
|
+
config.oauth_token = @oauth_token
|
61
|
+
config.oauth_token_secret = @oauth_token_secret
|
62
|
+
end
|
63
|
+
|
64
|
+
client = ::Twitter::Client.new
|
65
|
+
client.update(message)
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
module Backup
|
4
|
+
class Package
|
5
|
+
|
6
|
+
##
|
7
|
+
# The time when the backup initiated (in format: 2011.02.20.03.29.59)
|
8
|
+
attr_reader :time
|
9
|
+
|
10
|
+
##
|
11
|
+
# The trigger which initiated the backup process
|
12
|
+
attr_reader :trigger
|
13
|
+
|
14
|
+
##
|
15
|
+
# Extension for the final archive file(s)
|
16
|
+
attr_accessor :extension
|
17
|
+
|
18
|
+
##
|
19
|
+
# Set by the Splitter if the final archive was "chunked"
|
20
|
+
attr_accessor :chunk_suffixes
|
21
|
+
|
22
|
+
##
|
23
|
+
# The version of Backup used to create the package
|
24
|
+
attr_reader :version
|
25
|
+
|
26
|
+
def initialize(model)
|
27
|
+
@time = model.time
|
28
|
+
@trigger = model.trigger
|
29
|
+
@extension = 'tar'
|
30
|
+
@chunk_suffixes = Array.new
|
31
|
+
@version = Backup::Version.current
|
32
|
+
end
|
33
|
+
|
34
|
+
def filenames
|
35
|
+
if chunk_suffixes.empty?
|
36
|
+
[basename]
|
37
|
+
else
|
38
|
+
chunk_suffixes.map {|suffix| "#{ basename }-#{ suffix }" }
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def basename
|
43
|
+
"#{ time }.#{ trigger }.#{ extension }"
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,100 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
module Backup
|
4
|
+
module Packager
|
5
|
+
class << self
|
6
|
+
include Backup::CLI::Helpers
|
7
|
+
|
8
|
+
##
|
9
|
+
# Build the final package for the backup model.
|
10
|
+
def package!(model)
|
11
|
+
@package = model.package
|
12
|
+
@encryptor = model.encryptor
|
13
|
+
@splitter = model.splitter
|
14
|
+
@pipeline = Pipeline.new
|
15
|
+
|
16
|
+
Logger.message "Packaging the backup files..."
|
17
|
+
procedure.call
|
18
|
+
|
19
|
+
if @pipeline.success?
|
20
|
+
Logger.message "Packaging Complete!"
|
21
|
+
else
|
22
|
+
raise Errors::Packager::PipelineError,
|
23
|
+
"Failed to Create Backup Package\n" +
|
24
|
+
@pipeline.error_messages
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
##
|
31
|
+
# Builds a chain of nested Procs which adds each command to a Pipeline
|
32
|
+
# needed to package the final command to package the backup.
|
33
|
+
# This is done so that the Encryptor and Splitter have the ability
|
34
|
+
# to perform actions before and after the final command is executed.
|
35
|
+
# No Encryptors currently utilize this, however the Splitter does.
|
36
|
+
def procedure
|
37
|
+
stack = []
|
38
|
+
|
39
|
+
##
|
40
|
+
# Initial `tar` command to package the temporary backup folder.
|
41
|
+
# The command's output will then be either piped to the Encryptor
|
42
|
+
# or the Splitter (if no Encryptor), or through `cat` into the final
|
43
|
+
# output file if neither are configured.
|
44
|
+
@pipeline << "#{ utility(:tar) } -cf - " +
|
45
|
+
"-C '#{ Config.tmp_path }' '#{ @package.trigger }'"
|
46
|
+
|
47
|
+
##
|
48
|
+
# If an Encryptor was configured, it will be called first
|
49
|
+
# to add the encryption utility command to be piped through,
|
50
|
+
# and amend the final package extension.
|
51
|
+
# It's output will then be either piped into a Splitter,
|
52
|
+
# or through `cat` into the final output file.
|
53
|
+
if @encryptor
|
54
|
+
stack << lambda do
|
55
|
+
@encryptor.encrypt_with do |command, ext|
|
56
|
+
@pipeline << command
|
57
|
+
@package.extension << ext
|
58
|
+
stack.shift.call
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
##
|
64
|
+
# If a Splitter was configured, the `split` utility command will be
|
65
|
+
# added to the Pipeline to split the final output into multiple files.
|
66
|
+
# Once the Proc executing the Pipeline has completed and returns back
|
67
|
+
# to the Splitter, it will check the final output files to determine
|
68
|
+
# if the backup was indeed split.
|
69
|
+
# If so, it will set the package's chunk_suffixes. If not, it will
|
70
|
+
# remove the '-aa' suffix from the only file created by `split`.
|
71
|
+
#
|
72
|
+
# If no Splitter was configured, the final file output will be
|
73
|
+
# piped through `cat` into the final output file.
|
74
|
+
if @splitter
|
75
|
+
stack << lambda do
|
76
|
+
@splitter.split_with do |command|
|
77
|
+
@pipeline << command
|
78
|
+
stack.shift.call
|
79
|
+
end
|
80
|
+
end
|
81
|
+
else
|
82
|
+
stack << lambda do
|
83
|
+
outfile = File.join(Config.tmp_path, @package.basename)
|
84
|
+
@pipeline << "cat > #{ outfile }"
|
85
|
+
stack.shift.call
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
##
|
90
|
+
# Last Proc to be called runs the Pipeline the procedure built.
|
91
|
+
# Once complete, the call stack will unwind back through the
|
92
|
+
# preceeding Procs in the stack (if any)
|
93
|
+
stack << lambda { @pipeline.run }
|
94
|
+
|
95
|
+
stack.shift
|
96
|
+
end
|
97
|
+
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
@@ -0,0 +1,110 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
module Backup
|
4
|
+
class Pipeline
|
5
|
+
include Backup::CLI::Helpers
|
6
|
+
|
7
|
+
attr_reader :stderr, :errors
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@commands = []
|
11
|
+
@errors = []
|
12
|
+
@stderr = ''
|
13
|
+
end
|
14
|
+
|
15
|
+
##
|
16
|
+
# Adds a command to be executed in the pipeline.
|
17
|
+
# Each command will be run in the order in which it was added,
|
18
|
+
# with it's output being piped to the next command.
|
19
|
+
def <<(command)
|
20
|
+
@commands << command
|
21
|
+
end
|
22
|
+
|
23
|
+
##
|
24
|
+
# Runs the command line from `#pipeline` and collects STDOUT/STDERR.
|
25
|
+
# STDOUT is then parsed to determine the exit status of each command.
|
26
|
+
# For each command with a non-zero exit status, a SystemCallError is
|
27
|
+
# created and added to @errors. All STDERR output is set in @stderr.
|
28
|
+
#
|
29
|
+
# Note that there is no accumulated STDOUT from the commands themselves.
|
30
|
+
# Also, the last command should not attempt to write to STDOUT.
|
31
|
+
# Any output on STDOUT from the final command will be sent to STDERR.
|
32
|
+
# This in itself will not cause #run to fail, but will log warnings
|
33
|
+
# when all commands exit with non-zero status.
|
34
|
+
#
|
35
|
+
# Use `#success?` to determine if all commands in the pipeline succeeded.
|
36
|
+
# If `#success?` returns `false`, use `#error_messages` to get an error report.
|
37
|
+
def run
|
38
|
+
Open4.popen4(pipeline) do |pid, stdin, stdout, stderr|
|
39
|
+
pipestatus = stdout.read.gsub("\n", '').split(':').sort
|
40
|
+
pipestatus.each do |status|
|
41
|
+
index, exitstatus = status.split('|').map(&:to_i)
|
42
|
+
if exitstatus > 0
|
43
|
+
command = command_name(@commands[index])
|
44
|
+
@errors << SystemCallError.new(
|
45
|
+
"'#{ command }' returned exit code: #{ exitstatus }", exitstatus
|
46
|
+
)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
@stderr = stderr.read.strip
|
50
|
+
end
|
51
|
+
Logger.warn(stderr_messages) if success? && stderr_messages
|
52
|
+
rescue Exception => e
|
53
|
+
raise Errors::Pipeline::ExecutionError.wrap(e)
|
54
|
+
end
|
55
|
+
|
56
|
+
def success?
|
57
|
+
@errors.empty?
|
58
|
+
end
|
59
|
+
|
60
|
+
##
|
61
|
+
# Returns a multi-line String, reporting all STDERR messages received
|
62
|
+
# from the commands in the pipeline (if any), along with the SystemCallError
|
63
|
+
# (Errno) message for each command which had a non-zero exit status.
|
64
|
+
#
|
65
|
+
# Each error is wrapped by Backup::Errors to provide formatting.
|
66
|
+
def error_messages
|
67
|
+
@error_messages ||= (stderr_messages || '') +
|
68
|
+
"The following system errors were returned:\n" +
|
69
|
+
@errors.map {|err| Errors::Error.wrap(err).message }.join("\n")
|
70
|
+
end
|
71
|
+
|
72
|
+
private
|
73
|
+
|
74
|
+
##
|
75
|
+
# Each command is added as part of the pipeline, grouped with an `echo`
|
76
|
+
# command to pass along the command's index in @commands and it's exit status.
|
77
|
+
# The command's STDERR is redirected to FD#4, and the `echo` command to
|
78
|
+
# report the "index|exit status" is redirected to FD#3.
|
79
|
+
# Each command's STDOUT will be connected to the STDIN of the next subshell.
|
80
|
+
# The entire pipeline is run within a container group, which redirects
|
81
|
+
# FD#3 to STDOUT and FD#4 to STDERR so these can be collected.
|
82
|
+
# FD#1 is redirected to STDERR so that any output from the final command
|
83
|
+
# on STDOUT will generate warnings, since the final command should not
|
84
|
+
# attempt to write to STDOUT, as this would interfere with collecting
|
85
|
+
# the exit statuses.
|
86
|
+
#
|
87
|
+
# There is no guarantee as to the order of this output, which is why the
|
88
|
+
# command's index in @commands is passed along with it's exit status.
|
89
|
+
# And, if multiple commands output messages on STDERR, those messages
|
90
|
+
# may be interleaved. Interleaving of the "index|exit status" outputs
|
91
|
+
# should not be an issue, given the small byte size of the data being written.
|
92
|
+
def pipeline
|
93
|
+
parts = []
|
94
|
+
@commands.each_with_index do |command, index|
|
95
|
+
parts << %Q[{ #{ command } 2>&4 ; echo "#{ index }|$?:" >&3 ; }]
|
96
|
+
end
|
97
|
+
%Q[{ #{ parts.join(' | ') } } 3>&1 1>&2 4>&2]
|
98
|
+
end
|
99
|
+
|
100
|
+
def stderr_messages
|
101
|
+
@stderr_messages ||= @stderr.empty? ? false : <<-EOS.gsub(/^ +/, ' ')
|
102
|
+
Pipeline STDERR Messages:
|
103
|
+
(Note: may be interleaved if multiple commands returned error messages)
|
104
|
+
|
105
|
+
#{ @stderr }
|
106
|
+
EOS
|
107
|
+
end
|
108
|
+
|
109
|
+
end
|
110
|
+
end
|