backup 4.3.0 → 5.0.0.beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/LICENSE +19 -0
- data/README.md +13 -9
- data/bin/docker_test +24 -0
- data/lib/backup.rb +74 -78
- data/lib/backup/archive.rb +31 -32
- data/lib/backup/binder.rb +2 -6
- data/lib/backup/cleaner.rb +14 -18
- data/lib/backup/cli.rb +104 -108
- data/lib/backup/cloud_io/base.rb +4 -7
- data/lib/backup/cloud_io/cloud_files.rb +60 -62
- data/lib/backup/cloud_io/s3.rb +69 -76
- data/lib/backup/compressor/base.rb +4 -7
- data/lib/backup/compressor/bzip2.rb +3 -7
- data/lib/backup/compressor/custom.rb +2 -6
- data/lib/backup/compressor/gzip.rb +16 -17
- data/lib/backup/config.rb +17 -18
- data/lib/backup/config/dsl.rb +16 -17
- data/lib/backup/config/helpers.rb +10 -16
- data/lib/backup/database/base.rb +22 -21
- data/lib/backup/database/mongodb.rb +36 -37
- data/lib/backup/database/mysql.rb +40 -41
- data/lib/backup/database/openldap.rb +8 -10
- data/lib/backup/database/postgresql.rb +29 -30
- data/lib/backup/database/redis.rb +27 -30
- data/lib/backup/database/riak.rb +15 -18
- data/lib/backup/database/sqlite.rb +4 -6
- data/lib/backup/encryptor/base.rb +2 -4
- data/lib/backup/encryptor/gpg.rb +49 -59
- data/lib/backup/encryptor/open_ssl.rb +11 -14
- data/lib/backup/errors.rb +7 -12
- data/lib/backup/logger.rb +16 -18
- data/lib/backup/logger/console.rb +5 -8
- data/lib/backup/logger/fog_adapter.rb +2 -6
- data/lib/backup/logger/logfile.rb +10 -12
- data/lib/backup/logger/syslog.rb +2 -4
- data/lib/backup/model.rb +33 -40
- data/lib/backup/notifier/base.rb +24 -26
- data/lib/backup/notifier/campfire.rb +9 -11
- data/lib/backup/notifier/command.rb +3 -3
- data/lib/backup/notifier/datadog.rb +9 -12
- data/lib/backup/notifier/flowdock.rb +13 -17
- data/lib/backup/notifier/hipchat.rb +18 -14
- data/lib/backup/notifier/http_post.rb +11 -14
- data/lib/backup/notifier/mail.rb +42 -54
- data/lib/backup/notifier/nagios.rb +5 -9
- data/lib/backup/notifier/pagerduty.rb +10 -12
- data/lib/backup/notifier/prowl.rb +15 -15
- data/lib/backup/notifier/pushover.rb +7 -10
- data/lib/backup/notifier/ses.rb +52 -17
- data/lib/backup/notifier/slack.rb +39 -40
- data/lib/backup/notifier/twitter.rb +2 -5
- data/lib/backup/notifier/zabbix.rb +11 -14
- data/lib/backup/package.rb +5 -9
- data/lib/backup/packager.rb +16 -17
- data/lib/backup/pipeline.rb +17 -21
- data/lib/backup/splitter.rb +8 -11
- data/lib/backup/storage/base.rb +5 -8
- data/lib/backup/storage/cloud_files.rb +21 -23
- data/lib/backup/storage/cycler.rb +10 -15
- data/lib/backup/storage/dropbox.rb +15 -21
- data/lib/backup/storage/ftp.rb +14 -10
- data/lib/backup/storage/local.rb +5 -8
- data/lib/backup/storage/qiniu.rb +8 -8
- data/lib/backup/storage/rsync.rb +24 -26
- data/lib/backup/storage/s3.rb +27 -28
- data/lib/backup/storage/scp.rb +10 -12
- data/lib/backup/storage/sftp.rb +10 -12
- data/lib/backup/syncer/base.rb +5 -8
- data/lib/backup/syncer/cloud/base.rb +27 -30
- data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
- data/lib/backup/syncer/cloud/local_file.rb +5 -8
- data/lib/backup/syncer/cloud/s3.rb +23 -24
- data/lib/backup/syncer/rsync/base.rb +6 -10
- data/lib/backup/syncer/rsync/local.rb +1 -5
- data/lib/backup/syncer/rsync/pull.rb +6 -10
- data/lib/backup/syncer/rsync/push.rb +18 -22
- data/lib/backup/template.rb +9 -14
- data/lib/backup/utilities.rb +78 -69
- data/lib/backup/version.rb +1 -3
- metadata +107 -677
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Compressor
|
5
3
|
class Base
|
@@ -18,18 +16,17 @@ module Backup
|
|
18
16
|
##
|
19
17
|
# Return the compressor name, with Backup namespace removed
|
20
18
|
def compressor_name
|
21
|
-
self.class.to_s.sub(
|
19
|
+
self.class.to_s.sub("Backup::", "")
|
22
20
|
end
|
23
21
|
|
24
22
|
##
|
25
23
|
# Logs a message to the console and log file to inform
|
26
24
|
# the client that Backup is using the compressor
|
27
25
|
def log!
|
28
|
-
Logger.info "Using #{
|
29
|
-
" Command: '#{
|
30
|
-
" Ext: '#{
|
26
|
+
Logger.info "Using #{compressor_name} for compression.\n" \
|
27
|
+
" Command: '#{@cmd}'\n" \
|
28
|
+
" Ext: '#{@ext}'"
|
31
29
|
end
|
32
|
-
|
33
30
|
end
|
34
31
|
end
|
35
32
|
end
|
@@ -1,9 +1,6 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Compressor
|
5
3
|
class Bzip2 < Base
|
6
|
-
|
7
4
|
##
|
8
5
|
# Specify the level of compression to use.
|
9
6
|
#
|
@@ -24,16 +21,15 @@ module Backup
|
|
24
21
|
|
25
22
|
instance_eval(&block) if block_given?
|
26
23
|
|
27
|
-
@cmd = "#{
|
28
|
-
@ext =
|
24
|
+
@cmd = "#{utility(:bzip2)}#{options}"
|
25
|
+
@ext = ".bz2"
|
29
26
|
end
|
30
27
|
|
31
28
|
private
|
32
29
|
|
33
30
|
def options
|
34
|
-
" -#{
|
31
|
+
" -#{@level}" if @level
|
35
32
|
end
|
36
|
-
|
37
33
|
end
|
38
34
|
end
|
39
35
|
end
|
@@ -1,9 +1,6 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Compressor
|
5
3
|
class Custom < Base
|
6
|
-
|
7
4
|
##
|
8
5
|
# Specify the system command to invoke a compressor,
|
9
6
|
# including any command-line arguments.
|
@@ -36,9 +33,9 @@ module Backup
|
|
36
33
|
# Return the command line using the full path.
|
37
34
|
# Ensures the command exists and is executable.
|
38
35
|
def set_cmd
|
39
|
-
parts = @command.to_s.split(
|
36
|
+
parts = @command.to_s.split(" ")
|
40
37
|
parts[0] = utility(parts[0])
|
41
|
-
parts.join(
|
38
|
+
parts.join(" ")
|
42
39
|
end
|
43
40
|
|
44
41
|
##
|
@@ -47,7 +44,6 @@ module Backup
|
|
47
44
|
def set_ext
|
48
45
|
@extension.to_s.strip
|
49
46
|
end
|
50
|
-
|
51
47
|
end
|
52
48
|
end
|
53
49
|
end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Compressor
|
5
3
|
class Gzip < Base
|
@@ -35,8 +33,8 @@ module Backup
|
|
35
33
|
# Determine if +--rsyncable+ is supported and cache the result.
|
36
34
|
def self.has_rsyncable?
|
37
35
|
return @has_rsyncable unless @has_rsyncable.nil?
|
38
|
-
cmd = "#{
|
39
|
-
@has_rsyncable =
|
36
|
+
cmd = "#{utility(:gzip)} --rsyncable --version >/dev/null 2>&1; echo $?"
|
37
|
+
@has_rsyncable = `#{cmd}`.chomp == "0"
|
40
38
|
end
|
41
39
|
|
42
40
|
##
|
@@ -49,26 +47,27 @@ module Backup
|
|
49
47
|
|
50
48
|
instance_eval(&block) if block_given?
|
51
49
|
|
52
|
-
@cmd = "#{
|
53
|
-
@ext =
|
50
|
+
@cmd = "#{utility(:gzip)}#{options}"
|
51
|
+
@ext = ".gz"
|
54
52
|
end
|
55
53
|
|
56
54
|
private
|
57
55
|
|
58
56
|
def options
|
59
|
-
opts =
|
60
|
-
opts << " -#{
|
61
|
-
if
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
57
|
+
opts = ""
|
58
|
+
opts << " -#{@level}" if @level
|
59
|
+
if @rsyncable
|
60
|
+
if self.class.has_rsyncable?
|
61
|
+
opts << " --rsyncable"
|
62
|
+
else
|
63
|
+
Logger.warn Error.new(<<-EOS)
|
64
|
+
'rsyncable' option ignored.
|
65
|
+
Your system's 'gzip' does not support the `--rsyncable` option.
|
66
|
+
EOS
|
67
|
+
end
|
68
|
+
end
|
69
69
|
opts
|
70
70
|
end
|
71
|
-
|
72
71
|
end
|
73
72
|
end
|
74
73
|
end
|
data/lib/backup/config.rb
CHANGED
@@ -1,15 +1,14 @@
|
|
1
|
-
|
2
|
-
require
|
3
|
-
require 'backup/config/helpers'
|
1
|
+
require "backup/config/dsl"
|
2
|
+
require "backup/config/helpers"
|
4
3
|
|
5
4
|
module Backup
|
6
5
|
module Config
|
7
6
|
class Error < Backup::Error; end
|
8
7
|
|
9
8
|
DEFAULTS = {
|
10
|
-
:
|
11
|
-
:
|
12
|
-
:
|
9
|
+
config_file: "config.rb",
|
10
|
+
data_path: ".data",
|
11
|
+
tmp_path: ".tmp"
|
13
12
|
}
|
14
13
|
|
15
14
|
class << self
|
@@ -19,20 +18,20 @@ module Backup
|
|
19
18
|
|
20
19
|
# Loads the user's +config.rb+ and all model files.
|
21
20
|
def load(options = {})
|
22
|
-
update(options)
|
21
|
+
update(options) # from the command line
|
23
22
|
|
24
23
|
unless File.exist?(config_file)
|
25
24
|
raise Error, "Could not find configuration file: '#{config_file}'."
|
26
25
|
end
|
27
26
|
|
28
27
|
config = File.read(config_file)
|
29
|
-
version = Backup::VERSION.split(
|
28
|
+
version = Backup::VERSION.split(".").first
|
30
29
|
unless config =~ /^# Backup v#{ version }\.x Configuration$/
|
31
30
|
raise Error, <<-EOS
|
32
31
|
Invalid Configuration File
|
33
|
-
The configuration file at '#{
|
34
|
-
does not appear to be a Backup v#{
|
35
|
-
If you have upgraded to v#{
|
32
|
+
The configuration file at '#{config_file}'
|
33
|
+
does not appear to be a Backup v#{version}.x configuration file.
|
34
|
+
If you have upgraded to v#{version}.x from a previous version,
|
36
35
|
you need to upgrade your configuration file.
|
37
36
|
Please see the instructions for upgrading in the Backup documentation.
|
38
37
|
EOS
|
@@ -44,7 +43,7 @@ module Backup
|
|
44
43
|
update(dsl._config_options) # from config.rb
|
45
44
|
update(options) # command line takes precedence
|
46
45
|
|
47
|
-
Dir[File.join(File.dirname(config_file),
|
46
|
+
Dir[File.join(File.dirname(config_file), "models", "*.rb")].each do |model|
|
48
47
|
dsl.instance_eval(File.read(model), model)
|
49
48
|
end
|
50
49
|
end
|
@@ -79,7 +78,7 @@ module Backup
|
|
79
78
|
raise Error, <<-EOS
|
80
79
|
Root Path Not Found
|
81
80
|
When specifying a --root-path, the path must exist.
|
82
|
-
Path was: #{
|
81
|
+
Path was: #{path}
|
83
82
|
EOS
|
84
83
|
end
|
85
84
|
@root_path = path
|
@@ -88,7 +87,7 @@ module Backup
|
|
88
87
|
def set_path_variable(name, path, ending, root_path)
|
89
88
|
# strip any trailing '/' in case the user supplied this as part of
|
90
89
|
# an absolute path, so we can match it against File.expand_path()
|
91
|
-
path = path.to_s.sub(/\/\s*$/,
|
90
|
+
path = path.to_s.sub(/\/\s*$/, "").lstrip
|
92
91
|
new_path = false
|
93
92
|
# If no path is given, the variable will not be set/updated
|
94
93
|
# unless a root_path was given. In which case the value will
|
@@ -108,12 +107,12 @@ module Backup
|
|
108
107
|
end
|
109
108
|
|
110
109
|
def reset!
|
111
|
-
@user = ENV[
|
112
|
-
@root_path = File.join(File.expand_path(ENV[
|
113
|
-
update(:
|
110
|
+
@user = ENV["USER"] || Etc.getpwuid.name
|
111
|
+
@root_path = File.join(File.expand_path(ENV["HOME"] || ""), "Backup")
|
112
|
+
update(root_path: @root_path)
|
114
113
|
end
|
115
114
|
end
|
116
115
|
|
117
|
-
reset!
|
116
|
+
reset! # set defaults on load
|
118
117
|
end
|
119
118
|
end
|
data/lib/backup/config/dsl.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Config
|
5
3
|
# Context for loading user config.rb and model files.
|
@@ -27,23 +25,23 @@ module Backup
|
|
27
25
|
create_modules(
|
28
26
|
DSL,
|
29
27
|
[ # Databases
|
30
|
-
[
|
28
|
+
["MySQL", "PostgreSQL", "MongoDB", "Redis", "Riak", "OpenLDAP", "SQLite"],
|
31
29
|
# Storages
|
32
|
-
[
|
33
|
-
|
30
|
+
["S3", "CloudFiles", "Dropbox", "FTP",
|
31
|
+
"SFTP", "SCP", "RSync", "Local", "Qiniu"],
|
34
32
|
# Compressors
|
35
|
-
[
|
33
|
+
["Gzip", "Bzip2", "Custom"],
|
36
34
|
# Encryptors
|
37
|
-
[
|
35
|
+
["OpenSSL", "GPG"],
|
38
36
|
# Syncers
|
39
37
|
[
|
40
|
-
{
|
41
|
-
{
|
38
|
+
{ "Cloud" => ["CloudFiles", "S3"] },
|
39
|
+
{ "RSync" => ["Push", "Pull", "Local"] }
|
42
40
|
],
|
43
41
|
# Notifiers
|
44
|
-
[
|
45
|
-
|
46
|
-
|
42
|
+
["Mail", "Twitter", "Campfire", "Prowl",
|
43
|
+
"Hipchat", "PagerDuty", "Pushover", "HttpPost", "Nagios",
|
44
|
+
"Slack", "FlowDock", "Zabbix", "Ses", "DataDog", "Command"]
|
47
45
|
]
|
48
46
|
)
|
49
47
|
end
|
@@ -69,7 +67,7 @@ module Backup
|
|
69
67
|
end
|
70
68
|
end
|
71
69
|
|
72
|
-
add_dsl_constants
|
70
|
+
add_dsl_constants # add constants on load
|
73
71
|
|
74
72
|
attr_reader :_config_options
|
75
73
|
|
@@ -79,25 +77,26 @@ module Backup
|
|
79
77
|
|
80
78
|
# Allow users to set command line path options in config.rb
|
81
79
|
[:root_path, :data_path, :tmp_path].each do |name|
|
82
|
-
define_method name
|
80
|
+
define_method name do |path|
|
81
|
+
_config_options[name] = path
|
82
|
+
end
|
83
83
|
end
|
84
84
|
|
85
85
|
# Allows users to create preconfigured models.
|
86
86
|
def preconfigure(name, &block)
|
87
87
|
unless name.is_a?(String) && name =~ /^[A-Z]/
|
88
|
-
raise Error, "Preconfigured model names must be given as a string "
|
88
|
+
raise Error, "Preconfigured model names must be given as a string " \
|
89
89
|
"and start with a capital letter."
|
90
90
|
end
|
91
91
|
|
92
92
|
if DSL.const_defined?(name)
|
93
|
-
raise Error, "'#{
|
93
|
+
raise Error, "'#{name}' is already in use " \
|
94
94
|
"and can not be used for a preconfigured model."
|
95
95
|
end
|
96
96
|
|
97
97
|
DSL.const_set(name, Class.new(Model))
|
98
98
|
DSL.const_get(name).preconfigure(&block)
|
99
99
|
end
|
100
|
-
|
101
100
|
end
|
102
101
|
end
|
103
102
|
end
|
@@ -1,16 +1,13 @@
|
|
1
|
-
|
2
|
-
require 'ostruct'
|
1
|
+
require "ostruct"
|
3
2
|
|
4
3
|
module Backup
|
5
4
|
module Config
|
6
5
|
module Helpers
|
7
|
-
|
8
6
|
def self.included(klass)
|
9
7
|
klass.extend ClassMethods
|
10
8
|
end
|
11
9
|
|
12
10
|
module ClassMethods
|
13
|
-
|
14
11
|
def defaults
|
15
12
|
@defaults ||= Config::Defaults.new
|
16
13
|
|
@@ -31,12 +28,12 @@ module Backup
|
|
31
28
|
end
|
32
29
|
|
33
30
|
def log_deprecation_warning(name, deprecation)
|
34
|
-
msg = "#{
|
35
|
-
"backup v.#{
|
36
|
-
msg << "\n#{
|
31
|
+
msg = "#{self}##{name} has been deprecated as of " \
|
32
|
+
"backup v.#{deprecation[:version]}"
|
33
|
+
msg << "\n#{deprecation[:message]}" if deprecation[:message]
|
37
34
|
Logger.warn Config::Error.new(<<-EOS)
|
38
35
|
[DEPRECATION WARNING]
|
39
|
-
#{
|
36
|
+
#{msg}
|
40
37
|
EOS
|
41
38
|
end
|
42
39
|
|
@@ -65,12 +62,11 @@ module Backup
|
|
65
62
|
#
|
66
63
|
def attr_deprecate(name, args = {})
|
67
64
|
deprecations[name] = {
|
68
|
-
:
|
69
|
-
:
|
70
|
-
:
|
65
|
+
version: nil,
|
66
|
+
message: nil,
|
67
|
+
action: nil
|
71
68
|
}.merge(args)
|
72
69
|
end
|
73
|
-
|
74
70
|
end # ClassMethods
|
75
71
|
|
76
72
|
private
|
@@ -107,10 +103,10 @@ module Backup
|
|
107
103
|
#
|
108
104
|
def method_missing(name, *args)
|
109
105
|
deprecation = nil
|
110
|
-
if method = name.to_s.chomp!(
|
106
|
+
if method = name.to_s.chomp!("=")
|
111
107
|
if (len = args.count) != 1
|
112
108
|
raise ArgumentError,
|
113
|
-
"wrong number of arguments (#{
|
109
|
+
"wrong number of arguments (#{len} for 1)", caller(1)
|
114
110
|
end
|
115
111
|
deprecation = self.class.deprecations[method.to_sym]
|
116
112
|
end
|
@@ -122,7 +118,6 @@ module Backup
|
|
122
118
|
super
|
123
119
|
end
|
124
120
|
end
|
125
|
-
|
126
121
|
end # Helpers
|
127
122
|
|
128
123
|
# Store for pre-configured defaults.
|
@@ -138,6 +133,5 @@ module Backup
|
|
138
133
|
@table.clear
|
139
134
|
end
|
140
135
|
end
|
141
|
-
|
142
136
|
end
|
143
137
|
end
|
data/lib/backup/database/base.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Database
|
5
3
|
class Error < Backup::Error; end
|
@@ -16,8 +14,8 @@ module Backup
|
|
16
14
|
# the model.
|
17
15
|
def initialize(model, database_id = nil)
|
18
16
|
@model = model
|
19
|
-
@database_id = database_id.to_s.gsub(/\W/,
|
20
|
-
@dump_path = File.join(Config.tmp_path, model.trigger,
|
17
|
+
@database_id = database_id.to_s.gsub(/\W/, "_") if database_id
|
18
|
+
@dump_path = File.join(Config.tmp_path, model.trigger, "databases")
|
21
19
|
load_defaults!
|
22
20
|
end
|
23
21
|
|
@@ -45,11 +43,13 @@ module Backup
|
|
45
43
|
# Model#initialize calls this method *after* all defined databases have
|
46
44
|
# been initialized so `backup check` can report these warnings.
|
47
45
|
def dump_filename
|
48
|
-
@dump_filename ||=
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
46
|
+
@dump_filename ||=
|
47
|
+
begin
|
48
|
+
unless database_id
|
49
|
+
if model.databases.select { |d| d.class == self.class }.count > 1
|
50
|
+
sleep 1
|
51
|
+
@database_id = Time.now.to_i.to_s[-5, 5]
|
52
|
+
Logger.warn Error.new(<<-EOS)
|
53
53
|
Database Identifier Missing
|
54
54
|
When multiple Databases are configured in a single Backup Model
|
55
55
|
that have the same class (MySQL, PostgreSQL, etc.), the optional
|
@@ -58,27 +58,28 @@ module Backup
|
|
58
58
|
This will result in an output file in your final backup package like:
|
59
59
|
databases/MySQL-database_id.sql
|
60
60
|
|
61
|
-
Backup has auto-generated an identifier (#{
|
61
|
+
Backup has auto-generated an identifier (#{database_id}) for this
|
62
62
|
database dump and will now continue.
|
63
|
-
|
63
|
+
EOS
|
64
|
+
end
|
64
65
|
end
|
65
|
-
end
|
66
66
|
|
67
|
-
|
68
|
-
|
67
|
+
self.class.name.split("::").last + (database_id ? "-#{database_id}" : "")
|
68
|
+
end
|
69
69
|
end
|
70
70
|
|
71
71
|
def database_name
|
72
|
-
@database_name ||= self.class.to_s.sub(
|
73
|
-
|
72
|
+
@database_name ||= self.class.to_s.sub("Backup::", "") +
|
73
|
+
(database_id ? " (#{database_id})" : "")
|
74
74
|
end
|
75
75
|
|
76
76
|
def log!(action)
|
77
|
-
msg =
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
77
|
+
msg =
|
78
|
+
case action
|
79
|
+
when :started then "Started..."
|
80
|
+
when :finished then "Finished!"
|
81
|
+
end
|
82
|
+
Logger.info "#{database_name} #{msg}"
|
82
83
|
end
|
83
84
|
end
|
84
85
|
end
|