backedup 5.0.0.beta.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/LICENSE +19 -0
- data/README.md +33 -0
- data/bin/backedup +5 -0
- data/bin/docker_test +24 -0
- data/lib/backup/archive.rb +169 -0
- data/lib/backup/binder.rb +18 -0
- data/lib/backup/cleaner.rb +112 -0
- data/lib/backup/cli.rb +370 -0
- data/lib/backup/cloud_io/base.rb +38 -0
- data/lib/backup/cloud_io/cloud_files.rb +296 -0
- data/lib/backup/cloud_io/gcs.rb +121 -0
- data/lib/backup/cloud_io/s3.rb +253 -0
- data/lib/backup/cloud_io/swift.rb +96 -0
- data/lib/backup/compressor/base.rb +32 -0
- data/lib/backup/compressor/bzip2.rb +35 -0
- data/lib/backup/compressor/custom.rb +49 -0
- data/lib/backup/compressor/gzip.rb +73 -0
- data/lib/backup/compressor/pbzip2.rb +45 -0
- data/lib/backup/config/dsl.rb +102 -0
- data/lib/backup/config/helpers.rb +137 -0
- data/lib/backup/config.rb +118 -0
- data/lib/backup/database/base.rb +86 -0
- data/lib/backup/database/mongodb.rb +186 -0
- data/lib/backup/database/mysql.rb +191 -0
- data/lib/backup/database/openldap.rb +93 -0
- data/lib/backup/database/postgresql.rb +164 -0
- data/lib/backup/database/redis.rb +176 -0
- data/lib/backup/database/riak.rb +79 -0
- data/lib/backup/database/sqlite.rb +55 -0
- data/lib/backup/encryptor/base.rb +27 -0
- data/lib/backup/encryptor/gpg.rb +737 -0
- data/lib/backup/encryptor/open_ssl.rb +74 -0
- data/lib/backup/errors.rb +53 -0
- data/lib/backup/logger/console.rb +48 -0
- data/lib/backup/logger/fog_adapter.rb +25 -0
- data/lib/backup/logger/logfile.rb +131 -0
- data/lib/backup/logger/syslog.rb +114 -0
- data/lib/backup/logger.rb +197 -0
- data/lib/backup/model.rb +472 -0
- data/lib/backup/notifier/base.rb +126 -0
- data/lib/backup/notifier/campfire.rb +61 -0
- data/lib/backup/notifier/command.rb +99 -0
- data/lib/backup/notifier/datadog.rb +104 -0
- data/lib/backup/notifier/flowdock.rb +99 -0
- data/lib/backup/notifier/hipchat.rb +116 -0
- data/lib/backup/notifier/http_post.rb +114 -0
- data/lib/backup/notifier/mail.rb +232 -0
- data/lib/backup/notifier/nagios.rb +65 -0
- data/lib/backup/notifier/pagerduty.rb +79 -0
- data/lib/backup/notifier/prowl.rb +68 -0
- data/lib/backup/notifier/pushover.rb +71 -0
- data/lib/backup/notifier/ses.rb +123 -0
- data/lib/backup/notifier/slack.rb +147 -0
- data/lib/backup/notifier/twitter.rb +55 -0
- data/lib/backup/notifier/zabbix.rb +60 -0
- data/lib/backup/package.rb +51 -0
- data/lib/backup/packager.rb +106 -0
- data/lib/backup/pipeline.rb +120 -0
- data/lib/backup/splitter.rb +73 -0
- data/lib/backup/storage/base.rb +66 -0
- data/lib/backup/storage/cloud_files.rb +156 -0
- data/lib/backup/storage/cycler.rb +70 -0
- data/lib/backup/storage/dropbox.rb +206 -0
- data/lib/backup/storage/ftp.rb +116 -0
- data/lib/backup/storage/gcs.rb +93 -0
- data/lib/backup/storage/local.rb +61 -0
- data/lib/backup/storage/qiniu.rb +65 -0
- data/lib/backup/storage/rsync.rb +246 -0
- data/lib/backup/storage/s3.rb +155 -0
- data/lib/backup/storage/scp.rb +65 -0
- data/lib/backup/storage/sftp.rb +80 -0
- data/lib/backup/storage/swift.rb +124 -0
- data/lib/backup/storage/webdav.rb +102 -0
- data/lib/backup/syncer/base.rb +67 -0
- data/lib/backup/syncer/cloud/base.rb +176 -0
- data/lib/backup/syncer/cloud/cloud_files.rb +81 -0
- data/lib/backup/syncer/cloud/local_file.rb +97 -0
- data/lib/backup/syncer/cloud/s3.rb +109 -0
- data/lib/backup/syncer/rsync/base.rb +50 -0
- data/lib/backup/syncer/rsync/local.rb +27 -0
- data/lib/backup/syncer/rsync/pull.rb +47 -0
- data/lib/backup/syncer/rsync/push.rb +201 -0
- data/lib/backup/template.rb +41 -0
- data/lib/backup/utilities.rb +234 -0
- data/lib/backup/version.rb +3 -0
- data/lib/backup.rb +145 -0
- data/templates/cli/archive +28 -0
- data/templates/cli/compressor/bzip2 +4 -0
- data/templates/cli/compressor/custom +7 -0
- data/templates/cli/compressor/gzip +4 -0
- data/templates/cli/config +123 -0
- data/templates/cli/databases/mongodb +15 -0
- data/templates/cli/databases/mysql +18 -0
- data/templates/cli/databases/openldap +24 -0
- data/templates/cli/databases/postgresql +16 -0
- data/templates/cli/databases/redis +16 -0
- data/templates/cli/databases/riak +17 -0
- data/templates/cli/databases/sqlite +11 -0
- data/templates/cli/encryptor/gpg +27 -0
- data/templates/cli/encryptor/openssl +9 -0
- data/templates/cli/model +26 -0
- data/templates/cli/notifier/zabbix +15 -0
- data/templates/cli/notifiers/campfire +12 -0
- data/templates/cli/notifiers/command +32 -0
- data/templates/cli/notifiers/datadog +57 -0
- data/templates/cli/notifiers/flowdock +16 -0
- data/templates/cli/notifiers/hipchat +16 -0
- data/templates/cli/notifiers/http_post +32 -0
- data/templates/cli/notifiers/mail +24 -0
- data/templates/cli/notifiers/nagios +13 -0
- data/templates/cli/notifiers/pagerduty +12 -0
- data/templates/cli/notifiers/prowl +11 -0
- data/templates/cli/notifiers/pushover +11 -0
- data/templates/cli/notifiers/ses +15 -0
- data/templates/cli/notifiers/slack +22 -0
- data/templates/cli/notifiers/twitter +13 -0
- data/templates/cli/splitter +7 -0
- data/templates/cli/storages/cloud_files +11 -0
- data/templates/cli/storages/dropbox +20 -0
- data/templates/cli/storages/ftp +13 -0
- data/templates/cli/storages/gcs +8 -0
- data/templates/cli/storages/local +8 -0
- data/templates/cli/storages/qiniu +12 -0
- data/templates/cli/storages/rsync +17 -0
- data/templates/cli/storages/s3 +16 -0
- data/templates/cli/storages/scp +15 -0
- data/templates/cli/storages/sftp +15 -0
- data/templates/cli/storages/swift +19 -0
- data/templates/cli/storages/webdav +13 -0
- data/templates/cli/syncers/cloud_files +22 -0
- data/templates/cli/syncers/rsync_local +20 -0
- data/templates/cli/syncers/rsync_pull +28 -0
- data/templates/cli/syncers/rsync_push +28 -0
- data/templates/cli/syncers/s3 +27 -0
- data/templates/general/links +3 -0
- data/templates/general/version.erb +2 -0
- data/templates/notifier/mail/failure.erb +16 -0
- data/templates/notifier/mail/success.erb +16 -0
- data/templates/notifier/mail/warning.erb +16 -0
- data/templates/storage/dropbox/authorization_url.erb +6 -0
- data/templates/storage/dropbox/authorized.erb +4 -0
- data/templates/storage/dropbox/cache_file_written.erb +10 -0
- metadata +1255 -0
@@ -0,0 +1,147 @@
|
|
1
|
+
require "uri"
|
2
|
+
require "json"
|
3
|
+
|
4
|
+
module Backup
|
5
|
+
module Notifier
|
6
|
+
class Slack < Base
|
7
|
+
##
|
8
|
+
# The incoming webhook url
|
9
|
+
attr_accessor :webhook_url
|
10
|
+
|
11
|
+
##
|
12
|
+
# The channel to send messages to
|
13
|
+
attr_accessor :channel
|
14
|
+
|
15
|
+
##
|
16
|
+
# The username to display along with the notification
|
17
|
+
attr_accessor :username
|
18
|
+
|
19
|
+
##
|
20
|
+
# The emoji icon to display along with the notification
|
21
|
+
#
|
22
|
+
# See http://www.emoji-cheat-sheet.com for a list of icons.
|
23
|
+
#
|
24
|
+
# Default: :floppy_disk:
|
25
|
+
attr_accessor :icon_emoji
|
26
|
+
|
27
|
+
##
|
28
|
+
# Array of statuses for which the log file should be attached.
|
29
|
+
#
|
30
|
+
# Available statuses are: `:success`, `:warning` and `:failure`.
|
31
|
+
# Default: [:warning, :failure]
|
32
|
+
attr_accessor :send_log_on
|
33
|
+
|
34
|
+
def initialize(model, &block)
|
35
|
+
super
|
36
|
+
instance_eval(&block) if block_given?
|
37
|
+
|
38
|
+
@send_log_on ||= [:warning, :failure]
|
39
|
+
@icon_emoji ||= ":floppy_disk:"
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
##
|
45
|
+
# Notify the user of the backup operation results.
|
46
|
+
#
|
47
|
+
# `status` indicates one of the following:
|
48
|
+
#
|
49
|
+
# `:success`
|
50
|
+
# : The backup completed successfully.
|
51
|
+
# : Notification will be sent if `on_success` is `true`.
|
52
|
+
#
|
53
|
+
# `:warning`
|
54
|
+
# : The backup completed successfully, but warnings were logged.
|
55
|
+
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
56
|
+
#
|
57
|
+
# `:failure`
|
58
|
+
# : The backup operation failed.
|
59
|
+
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
60
|
+
#
|
61
|
+
def notify!(status)
|
62
|
+
data = {
|
63
|
+
text: message.call(model, status: status_data_for(status)),
|
64
|
+
attachments: [attachment(status)]
|
65
|
+
}
|
66
|
+
[:channel, :username, :icon_emoji].each do |param|
|
67
|
+
val = send(param)
|
68
|
+
data.merge!(param => val) if val
|
69
|
+
end
|
70
|
+
|
71
|
+
options = {
|
72
|
+
headers: { "Content-Type" => "application/x-www-form-urlencoded" },
|
73
|
+
body: URI.encode_www_form(payload: JSON.dump(data))
|
74
|
+
}
|
75
|
+
options[:expects] = 200 # raise error if unsuccessful
|
76
|
+
Excon.post(uri, options)
|
77
|
+
end
|
78
|
+
|
79
|
+
def attachment(status)
|
80
|
+
{
|
81
|
+
fallback: "#{title(status)} - Job: #{model.label} (#{model.trigger})",
|
82
|
+
text: title(status),
|
83
|
+
color: color(status),
|
84
|
+
fields: [
|
85
|
+
{
|
86
|
+
title: "Job",
|
87
|
+
value: "#{model.label} (#{model.trigger})",
|
88
|
+
short: false
|
89
|
+
},
|
90
|
+
{
|
91
|
+
title: "Started",
|
92
|
+
value: model.started_at,
|
93
|
+
short: true
|
94
|
+
},
|
95
|
+
{
|
96
|
+
title: "Finished",
|
97
|
+
value: model.finished_at,
|
98
|
+
short: true
|
99
|
+
},
|
100
|
+
{
|
101
|
+
title: "Duration",
|
102
|
+
value: model.duration,
|
103
|
+
short: true
|
104
|
+
},
|
105
|
+
{
|
106
|
+
title: "Version",
|
107
|
+
value: "Backup v#{Backup::VERSION}\nRuby: #{RUBY_DESCRIPTION}",
|
108
|
+
short: false
|
109
|
+
},
|
110
|
+
log_field(status)
|
111
|
+
].compact
|
112
|
+
}
|
113
|
+
end
|
114
|
+
|
115
|
+
def log_field(status)
|
116
|
+
send_log = send_log_on.include?(status)
|
117
|
+
return unless send_log
|
118
|
+
|
119
|
+
{
|
120
|
+
title: "Detailed Backup Log",
|
121
|
+
value: Logger.messages.map(&:formatted_lines).flatten.join("\n"),
|
122
|
+
short: false
|
123
|
+
}
|
124
|
+
end
|
125
|
+
|
126
|
+
def color(status)
|
127
|
+
case status
|
128
|
+
when :success then "good"
|
129
|
+
when :failure then "danger"
|
130
|
+
when :warning then "warning"
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
def title(status)
|
135
|
+
case status
|
136
|
+
when :success then "Backup Completed Successfully!"
|
137
|
+
when :failure then "Backup Failed!"
|
138
|
+
when :warning then "Backup Completed Successfully (with Warnings)!"
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def uri
|
143
|
+
@uri ||= webhook_url
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require "twitter"
|
2
|
+
|
3
|
+
module Backup
|
4
|
+
module Notifier
|
5
|
+
class Twitter < Base
|
6
|
+
##
|
7
|
+
# Twitter consumer key credentials
|
8
|
+
attr_accessor :consumer_key, :consumer_secret
|
9
|
+
|
10
|
+
##
|
11
|
+
# OAuth credentials
|
12
|
+
attr_accessor :oauth_token, :oauth_token_secret
|
13
|
+
|
14
|
+
def initialize(model, &block)
|
15
|
+
super
|
16
|
+
instance_eval(&block) if block_given?
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
##
|
22
|
+
# Notify the user of the backup operation results.
|
23
|
+
#
|
24
|
+
# `status` indicates one of the following:
|
25
|
+
#
|
26
|
+
# `:success`
|
27
|
+
# : The backup completed successfully.
|
28
|
+
# : Notification will be sent if `on_success` is `true`.
|
29
|
+
#
|
30
|
+
# `:warning`
|
31
|
+
# : The backup completed successfully, but warnings were logged.
|
32
|
+
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
33
|
+
#
|
34
|
+
# `:failure`
|
35
|
+
# : The backup operation failed.
|
36
|
+
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
37
|
+
#
|
38
|
+
def notify!(status)
|
39
|
+
send_message(message.call(model, status: status_data_for(status)))
|
40
|
+
end
|
41
|
+
|
42
|
+
# Twitter::Client will raise an error if unsuccessful.
|
43
|
+
def send_message(message)
|
44
|
+
client = ::Twitter::REST::Client.new do |config|
|
45
|
+
config.consumer_key = @consumer_key
|
46
|
+
config.consumer_secret = @consumer_secret
|
47
|
+
config.access_token = @oauth_token
|
48
|
+
config.access_token_secret = @oauth_token_secret
|
49
|
+
end
|
50
|
+
|
51
|
+
client.update(message)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
module Backup
|
2
|
+
module Notifier
|
3
|
+
class Zabbix < Base
|
4
|
+
attr_accessor :zabbix_host
|
5
|
+
|
6
|
+
attr_accessor :zabbix_port
|
7
|
+
|
8
|
+
attr_accessor :service_name
|
9
|
+
|
10
|
+
attr_accessor :service_host
|
11
|
+
|
12
|
+
attr_accessor :item_key
|
13
|
+
|
14
|
+
def initialize(model, &block)
|
15
|
+
super
|
16
|
+
instance_eval(&block) if block_given?
|
17
|
+
|
18
|
+
@zabbix_host ||= Config.hostname
|
19
|
+
@zabbix_port ||= 10_051
|
20
|
+
@service_name ||= "Backup #{model.trigger}"
|
21
|
+
@service_host ||= Config.hostname
|
22
|
+
@item_key ||= "backup_status"
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
##
|
28
|
+
# Notify the user of the backup operation results.
|
29
|
+
#
|
30
|
+
# `status` indicates one of the following:
|
31
|
+
#
|
32
|
+
# `:success`
|
33
|
+
# : The backup completed successfully.
|
34
|
+
# : Notification will be sent if `on_success` is `true`.
|
35
|
+
#
|
36
|
+
# `:warning`
|
37
|
+
# : The backup completed successfully, but warnings were logged.
|
38
|
+
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
39
|
+
#
|
40
|
+
# `:failure`
|
41
|
+
# : The backup operation failed.
|
42
|
+
# : Notification will be sent if `on_warning` or `on_success` is `true`.
|
43
|
+
#
|
44
|
+
def notify!(status)
|
45
|
+
send_message(message.call(model, status: status_data_for(status)))
|
46
|
+
end
|
47
|
+
|
48
|
+
def send_message(message)
|
49
|
+
msg = [service_host, service_name, model.exit_status, message].join("\t")
|
50
|
+
cmd = utility(:zabbix_sender).to_s +
|
51
|
+
" -z '#{zabbix_host}'" \
|
52
|
+
" -p '#{zabbix_port}'" \
|
53
|
+
" -s #{service_host}" \
|
54
|
+
" -k #{item_key}" \
|
55
|
+
" -o '#{msg}'"
|
56
|
+
run("echo '#{msg}' | #{cmd}")
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module Backup
|
2
|
+
class Package
|
3
|
+
##
|
4
|
+
# The time when the backup initiated (in format: 2011.02.20.03.29.59)
|
5
|
+
attr_accessor :time
|
6
|
+
|
7
|
+
##
|
8
|
+
# The trigger which initiated the backup process
|
9
|
+
attr_reader :trigger
|
10
|
+
|
11
|
+
##
|
12
|
+
# Extension for the final archive file(s)
|
13
|
+
attr_accessor :extension
|
14
|
+
|
15
|
+
##
|
16
|
+
# Set by the Splitter if the final archive was "chunked"
|
17
|
+
attr_accessor :chunk_suffixes
|
18
|
+
|
19
|
+
##
|
20
|
+
# If true, the Cycler will not attempt to remove the package when Cycling.
|
21
|
+
attr_accessor :no_cycle
|
22
|
+
|
23
|
+
##
|
24
|
+
# The version of Backup used to create the package
|
25
|
+
attr_reader :version
|
26
|
+
|
27
|
+
def initialize(model)
|
28
|
+
@trigger = model.trigger
|
29
|
+
@extension = "tar"
|
30
|
+
@chunk_suffixes = []
|
31
|
+
@no_cycle = false
|
32
|
+
@version = VERSION
|
33
|
+
end
|
34
|
+
|
35
|
+
def filenames
|
36
|
+
if chunk_suffixes.empty?
|
37
|
+
[basename]
|
38
|
+
else
|
39
|
+
chunk_suffixes.map { |suffix| "#{basename}-#{suffix}" }
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def basename
|
44
|
+
"#{trigger}.#{extension}"
|
45
|
+
end
|
46
|
+
|
47
|
+
def time_as_object
|
48
|
+
Time.strptime(time, "%Y.%m.%d.%H.%M.%S")
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
module Backup
|
2
|
+
module Packager
|
3
|
+
class Error < Backup::Error; end
|
4
|
+
|
5
|
+
class << self
|
6
|
+
include Utilities::Helpers
|
7
|
+
|
8
|
+
##
|
9
|
+
# Build the final package for the backup model.
|
10
|
+
def package!(model)
|
11
|
+
@package = model.package
|
12
|
+
@encryptor = model.encryptor
|
13
|
+
@splitter = model.splitter
|
14
|
+
@pipeline = Pipeline.new
|
15
|
+
|
16
|
+
Logger.info "Packaging the backup files..."
|
17
|
+
procedure.call
|
18
|
+
|
19
|
+
if @pipeline.success?
|
20
|
+
Logger.info "Packaging Complete!"
|
21
|
+
else
|
22
|
+
raise Error, "Failed to Create Backup Package\n" +
|
23
|
+
@pipeline.error_messages
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
##
|
30
|
+
# Builds a chain of nested Procs which adds each command to a Pipeline
|
31
|
+
# needed to package the final command to package the backup.
|
32
|
+
# This is done so that the Encryptor and Splitter have the ability
|
33
|
+
# to perform actions before and after the final command is executed.
|
34
|
+
# No Encryptors currently utilize this, however the Splitter does.
|
35
|
+
def procedure
|
36
|
+
stack = []
|
37
|
+
|
38
|
+
##
|
39
|
+
# Initial `tar` command to package the temporary backup folder.
|
40
|
+
# The command's output will then be either piped to the Encryptor
|
41
|
+
# or the Splitter (if no Encryptor), or through `cat` into the final
|
42
|
+
# output file if neither are configured.
|
43
|
+
@pipeline.add(
|
44
|
+
"#{utility(:tar)} -cf - " \
|
45
|
+
"-C '#{Config.tmp_path}' '#{@package.trigger}'",
|
46
|
+
tar_success_codes
|
47
|
+
)
|
48
|
+
|
49
|
+
##
|
50
|
+
# If an Encryptor was configured, it will be called first
|
51
|
+
# to add the encryption utility command to be piped through,
|
52
|
+
# and amend the final package extension.
|
53
|
+
# It's output will then be either piped into a Splitter,
|
54
|
+
# or through `cat` into the final output file.
|
55
|
+
if @encryptor
|
56
|
+
stack << lambda do
|
57
|
+
@encryptor.encrypt_with do |command, ext|
|
58
|
+
@pipeline << command
|
59
|
+
@package.extension << ext
|
60
|
+
stack.shift.call
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
##
|
66
|
+
# If a Splitter was configured, the `split` utility command will be
|
67
|
+
# added to the Pipeline to split the final output into multiple files.
|
68
|
+
# Once the Proc executing the Pipeline has completed and returns back
|
69
|
+
# to the Splitter, it will check the final output files to determine
|
70
|
+
# if the backup was indeed split.
|
71
|
+
# If so, it will set the package's chunk_suffixes. If not, it will
|
72
|
+
# remove the '-aa' suffix from the only file created by `split`.
|
73
|
+
#
|
74
|
+
# If no Splitter was configured, the final file output will be
|
75
|
+
# piped through `cat` into the final output file.
|
76
|
+
stack <<
|
77
|
+
if @splitter
|
78
|
+
lambda do
|
79
|
+
@splitter.split_with do |command|
|
80
|
+
@pipeline << command
|
81
|
+
stack.shift.call
|
82
|
+
end
|
83
|
+
end
|
84
|
+
else
|
85
|
+
lambda do
|
86
|
+
outfile = File.join(Config.tmp_path, @package.basename)
|
87
|
+
@pipeline << "#{utility(:cat)} > #{outfile}"
|
88
|
+
stack.shift.call
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
##
|
93
|
+
# Last Proc to be called runs the Pipeline the procedure built.
|
94
|
+
# Once complete, the call stack will unwind back through the
|
95
|
+
# preceeding Procs in the stack (if any)
|
96
|
+
stack << -> { @pipeline.run }
|
97
|
+
|
98
|
+
stack.shift
|
99
|
+
end
|
100
|
+
|
101
|
+
def tar_success_codes
|
102
|
+
gnu_tar? ? [0, 1] : [0]
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
module Backup
|
2
|
+
class Pipeline
|
3
|
+
class Error < Backup::Error; end
|
4
|
+
|
5
|
+
include Utilities::Helpers
|
6
|
+
|
7
|
+
attr_reader :stderr, :errors
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@commands = []
|
11
|
+
@success_codes = []
|
12
|
+
@errors = []
|
13
|
+
@stderr = ""
|
14
|
+
end
|
15
|
+
|
16
|
+
##
|
17
|
+
# Adds a command to be executed in the pipeline.
|
18
|
+
# Each command will be run in the order in which it was added,
|
19
|
+
# with it's output being piped to the next command.
|
20
|
+
#
|
21
|
+
# +success_codes+ must be an Array of Integer exit codes that will
|
22
|
+
# be considered successful for the +command+.
|
23
|
+
def add(command, success_codes)
|
24
|
+
@commands << command
|
25
|
+
@success_codes << success_codes
|
26
|
+
end
|
27
|
+
|
28
|
+
##
|
29
|
+
# Commands added using this method will only be considered successful
|
30
|
+
# if their exit status is 0.
|
31
|
+
#
|
32
|
+
# Use #add if successful exit status codes need to be specified.
|
33
|
+
def <<(command)
|
34
|
+
add(command, [0])
|
35
|
+
end
|
36
|
+
|
37
|
+
##
|
38
|
+
# Runs the command line from `#pipeline` and collects STDOUT/STDERR.
|
39
|
+
# STDOUT is then parsed to determine the exit status of each command.
|
40
|
+
# For each command with a non-zero exit status, a SystemCallError is
|
41
|
+
# created and added to @errors. All STDERR output is set in @stderr.
|
42
|
+
#
|
43
|
+
# Note that there is no accumulated STDOUT from the commands themselves.
|
44
|
+
# Also, the last command should not attempt to write to STDOUT.
|
45
|
+
# Any output on STDOUT from the final command will be sent to STDERR.
|
46
|
+
# This in itself will not cause #run to fail, but will log warnings
|
47
|
+
# when all commands exit with non-zero status.
|
48
|
+
#
|
49
|
+
# Use `#success?` to determine if all commands in the pipeline succeeded.
|
50
|
+
# If `#success?` returns `false`, use `#error_messages` to get an error report.
|
51
|
+
def run
|
52
|
+
Open4.popen4(pipeline) do |_pid, _stdin, stdout, stderr|
|
53
|
+
pipestatus = stdout.read.delete("\n").split(":").sort
|
54
|
+
pipestatus.each do |status|
|
55
|
+
index, exitstatus = status.split("|").map(&:to_i)
|
56
|
+
next if @success_codes[index].include?(exitstatus)
|
57
|
+
command = command_name(@commands[index])
|
58
|
+
@errors << SystemCallError.new(
|
59
|
+
"'#{command}' returned exit code: #{exitstatus}", exitstatus
|
60
|
+
)
|
61
|
+
end
|
62
|
+
@stderr = stderr.read.strip
|
63
|
+
end
|
64
|
+
Logger.warn(stderr_messages) if success? && stderr_messages
|
65
|
+
rescue Exception => err
|
66
|
+
raise Error.wrap(err, "Pipeline failed to execute")
|
67
|
+
end
|
68
|
+
|
69
|
+
def success?
|
70
|
+
@errors.empty?
|
71
|
+
end
|
72
|
+
|
73
|
+
##
|
74
|
+
# Returns a multi-line String, reporting all STDERR messages received
|
75
|
+
# from the commands in the pipeline (if any), along with the SystemCallError
|
76
|
+
# (Errno) message for each command which had a non-zero exit status.
|
77
|
+
def error_messages
|
78
|
+
@error_messages ||= (stderr_messages || "") +
|
79
|
+
"The following system errors were returned:\n" +
|
80
|
+
@errors.map { |err| "#{err.class}: #{err.message}" }.join("\n")
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
|
85
|
+
##
|
86
|
+
# Each command is added as part of the pipeline, grouped with an `echo`
|
87
|
+
# command to pass along the command's index in @commands and it's exit status.
|
88
|
+
# The command's STDERR is redirected to FD#4, and the `echo` command to
|
89
|
+
# report the "index|exit status" is redirected to FD#3.
|
90
|
+
# Each command's STDOUT will be connected to the STDIN of the next subshell.
|
91
|
+
# The entire pipeline is run within a container group, which redirects
|
92
|
+
# FD#3 to STDOUT and FD#4 to STDERR so these can be collected.
|
93
|
+
# FD#1 is redirected to STDERR so that any output from the final command
|
94
|
+
# on STDOUT will generate warnings, since the final command should not
|
95
|
+
# attempt to write to STDOUT, as this would interfere with collecting
|
96
|
+
# the exit statuses.
|
97
|
+
#
|
98
|
+
# There is no guarantee as to the order of this output, which is why the
|
99
|
+
# command's index in @commands is passed along with it's exit status.
|
100
|
+
# And, if multiple commands output messages on STDERR, those messages
|
101
|
+
# may be interleaved. Interleaving of the "index|exit status" outputs
|
102
|
+
# should not be an issue, given the small byte size of the data being written.
|
103
|
+
def pipeline
|
104
|
+
parts = []
|
105
|
+
@commands.each_with_index do |command, index|
|
106
|
+
parts << %({ #{command} 2>&4 ; echo "#{index}|$?:" >&3 ; })
|
107
|
+
end
|
108
|
+
%({ #{parts.join(" | ")} } 3>&1 1>&2 4>&2)
|
109
|
+
end
|
110
|
+
|
111
|
+
def stderr_messages
|
112
|
+
@stderr_messages ||= @stderr.empty? ? false : <<-EOS.gsub(/^ +/, " ")
|
113
|
+
Pipeline STDERR Messages:
|
114
|
+
(Note: may be interleaved if multiple commands returned error messages)
|
115
|
+
|
116
|
+
#{@stderr}
|
117
|
+
EOS
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
module Backup
|
2
|
+
class Splitter
|
3
|
+
include Utilities::Helpers
|
4
|
+
|
5
|
+
attr_reader :package, :chunk_size, :suffix_length
|
6
|
+
|
7
|
+
def initialize(model, chunk_size, suffix_length)
|
8
|
+
@package = model.package
|
9
|
+
@chunk_size = chunk_size
|
10
|
+
@suffix_length = suffix_length
|
11
|
+
end
|
12
|
+
|
13
|
+
##
|
14
|
+
# This is called as part of the procedure used to build the final
|
15
|
+
# backup package file(s). It yields it's portion of the command line
|
16
|
+
# for this procedure, which will split the data being piped into it
|
17
|
+
# into multiple files, based on the @chunk_size, using a suffix length as
|
18
|
+
# specified by @suffix_length.
|
19
|
+
# Once the packaging procedure is complete, it will return and
|
20
|
+
# @package.chunk_suffixes will be set based on the resulting files.
|
21
|
+
def split_with
|
22
|
+
Logger.info "Splitter configured with a chunk size of #{chunk_size}MB " \
|
23
|
+
"and suffix length of #{suffix_length}."
|
24
|
+
yield split_command
|
25
|
+
after_packaging
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
##
|
31
|
+
# The `split` command reads from $stdin and will store it's output in
|
32
|
+
# multiple files, based on @chunk_size and @suffix_length, using the full
|
33
|
+
# path to the final @package.basename, plus a '-' separator as the `prefix`.
|
34
|
+
def split_command
|
35
|
+
"#{utility(:split)} -a #{suffix_length} -b #{chunk_size}m - " \
|
36
|
+
"'#{File.join(Config.tmp_path, package.basename + "-")}'"
|
37
|
+
end
|
38
|
+
|
39
|
+
##
|
40
|
+
# Finds the resulting files from the packaging procedure
|
41
|
+
# and stores an Array of suffixes used in @package.chunk_suffixes.
|
42
|
+
# If the @chunk_size was never reached and only one file
|
43
|
+
# was written, that file will be suffixed with '-aa' (or -a; -aaa; etc
|
44
|
+
# depending upon suffix_length). In which case, it will simply
|
45
|
+
# remove the suffix from the filename.
|
46
|
+
def after_packaging
|
47
|
+
suffixes = chunk_suffixes
|
48
|
+
first_suffix = "a" * suffix_length
|
49
|
+
if suffixes == [first_suffix]
|
50
|
+
FileUtils.mv(
|
51
|
+
File.join(Config.tmp_path, "#{package.basename}-#{first_suffix}"),
|
52
|
+
File.join(Config.tmp_path, package.basename)
|
53
|
+
)
|
54
|
+
else
|
55
|
+
package.chunk_suffixes = suffixes
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
##
|
60
|
+
# Returns an array of suffixes for each chunk, in alphabetical order.
|
61
|
+
# For example: [aa, ab, ac, ad, ae] or [aaa, aab, aac aad]
|
62
|
+
def chunk_suffixes
|
63
|
+
chunks.map { |chunk| File.extname(chunk).split("-").last }.sort
|
64
|
+
end
|
65
|
+
|
66
|
+
##
|
67
|
+
# Returns an array of full paths to the backup chunks.
|
68
|
+
# Chunks are sorted in alphabetical order.
|
69
|
+
def chunks
|
70
|
+
Dir[File.join(Config.tmp_path, package.basename + "-*")].sort
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
module Backup
|
2
|
+
module Storage
|
3
|
+
class Base
|
4
|
+
include Config::Helpers
|
5
|
+
|
6
|
+
##
|
7
|
+
# Base path on the remote where backup package files will be stored.
|
8
|
+
attr_accessor :path
|
9
|
+
|
10
|
+
##
|
11
|
+
# Number of backups to keep or time until which to keep.
|
12
|
+
#
|
13
|
+
# If an Integer is given it sets the limit to how many backups to keep in
|
14
|
+
# the remote location. If exceeded, the oldest will be removed to make
|
15
|
+
# room for the newest.
|
16
|
+
#
|
17
|
+
# If a Time object is given it will remove backups _older_ than the given
|
18
|
+
# date.
|
19
|
+
#
|
20
|
+
# @!attribute [rw] keep
|
21
|
+
# @param [Integer|Time]
|
22
|
+
# @return [Integer|Time]
|
23
|
+
attr_accessor :keep
|
24
|
+
|
25
|
+
attr_reader :model, :package, :storage_id
|
26
|
+
|
27
|
+
##
|
28
|
+
# +storage_id+ is a user-defined string used to uniquely identify
|
29
|
+
# multiple storages of the same type. If multiple storages of the same
|
30
|
+
# type are added to a single backup model, this identifier must be set.
|
31
|
+
# This will be appended to the YAML storage file used for cycling backups.
|
32
|
+
def initialize(model, storage_id = nil, &block)
|
33
|
+
@model = model
|
34
|
+
@package = model.package
|
35
|
+
@storage_id = storage_id.to_s.gsub(/\W/, "_") if storage_id
|
36
|
+
|
37
|
+
load_defaults!
|
38
|
+
instance_eval(&block) if block_given?
|
39
|
+
end
|
40
|
+
|
41
|
+
def perform!
|
42
|
+
Logger.info "#{storage_name} Started..."
|
43
|
+
transfer!
|
44
|
+
if respond_to?(:cycle!, true) && (keep.to_i > 0 || keep.is_a?(Time))
|
45
|
+
cycle!
|
46
|
+
end
|
47
|
+
Logger.info "#{storage_name} Finished!"
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
##
|
53
|
+
# Return the remote path for the current or given package.
|
54
|
+
def remote_path(pkg = package)
|
55
|
+
path.empty? ? File.join(pkg.trigger, pkg.time) :
|
56
|
+
File.join(path, pkg.trigger, pkg.time)
|
57
|
+
end
|
58
|
+
alias :remote_path_for :remote_path
|
59
|
+
|
60
|
+
def storage_name
|
61
|
+
@storage_name ||= self.class.to_s.sub("Backup::", "") +
|
62
|
+
(storage_id ? " (#{storage_id})" : "")
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|