backup 4.3.0 → 5.0.0.beta.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/LICENSE +19 -0
- data/README.md +13 -9
- data/bin/docker_test +24 -0
- data/lib/backup.rb +74 -78
- data/lib/backup/archive.rb +31 -32
- data/lib/backup/binder.rb +2 -6
- data/lib/backup/cleaner.rb +14 -18
- data/lib/backup/cli.rb +104 -108
- data/lib/backup/cloud_io/base.rb +4 -7
- data/lib/backup/cloud_io/cloud_files.rb +60 -62
- data/lib/backup/cloud_io/s3.rb +69 -76
- data/lib/backup/compressor/base.rb +4 -7
- data/lib/backup/compressor/bzip2.rb +3 -7
- data/lib/backup/compressor/custom.rb +2 -6
- data/lib/backup/compressor/gzip.rb +16 -17
- data/lib/backup/config.rb +17 -18
- data/lib/backup/config/dsl.rb +16 -17
- data/lib/backup/config/helpers.rb +10 -16
- data/lib/backup/database/base.rb +22 -21
- data/lib/backup/database/mongodb.rb +36 -37
- data/lib/backup/database/mysql.rb +40 -41
- data/lib/backup/database/openldap.rb +8 -10
- data/lib/backup/database/postgresql.rb +29 -30
- data/lib/backup/database/redis.rb +27 -30
- data/lib/backup/database/riak.rb +15 -18
- data/lib/backup/database/sqlite.rb +4 -6
- data/lib/backup/encryptor/base.rb +2 -4
- data/lib/backup/encryptor/gpg.rb +49 -59
- data/lib/backup/encryptor/open_ssl.rb +11 -14
- data/lib/backup/errors.rb +7 -12
- data/lib/backup/logger.rb +16 -18
- data/lib/backup/logger/console.rb +5 -8
- data/lib/backup/logger/fog_adapter.rb +2 -6
- data/lib/backup/logger/logfile.rb +10 -12
- data/lib/backup/logger/syslog.rb +2 -4
- data/lib/backup/model.rb +33 -40
- data/lib/backup/notifier/base.rb +24 -26
- data/lib/backup/notifier/campfire.rb +9 -11
- data/lib/backup/notifier/command.rb +3 -3
- data/lib/backup/notifier/datadog.rb +9 -12
- data/lib/backup/notifier/flowdock.rb +13 -17
- data/lib/backup/notifier/hipchat.rb +18 -14
- data/lib/backup/notifier/http_post.rb +11 -14
- data/lib/backup/notifier/mail.rb +42 -54
- data/lib/backup/notifier/nagios.rb +5 -9
- data/lib/backup/notifier/pagerduty.rb +10 -12
- data/lib/backup/notifier/prowl.rb +15 -15
- data/lib/backup/notifier/pushover.rb +7 -10
- data/lib/backup/notifier/ses.rb +52 -17
- data/lib/backup/notifier/slack.rb +39 -40
- data/lib/backup/notifier/twitter.rb +2 -5
- data/lib/backup/notifier/zabbix.rb +11 -14
- data/lib/backup/package.rb +5 -9
- data/lib/backup/packager.rb +16 -17
- data/lib/backup/pipeline.rb +17 -21
- data/lib/backup/splitter.rb +8 -11
- data/lib/backup/storage/base.rb +5 -8
- data/lib/backup/storage/cloud_files.rb +21 -23
- data/lib/backup/storage/cycler.rb +10 -15
- data/lib/backup/storage/dropbox.rb +15 -21
- data/lib/backup/storage/ftp.rb +14 -10
- data/lib/backup/storage/local.rb +5 -8
- data/lib/backup/storage/qiniu.rb +8 -8
- data/lib/backup/storage/rsync.rb +24 -26
- data/lib/backup/storage/s3.rb +27 -28
- data/lib/backup/storage/scp.rb +10 -12
- data/lib/backup/storage/sftp.rb +10 -12
- data/lib/backup/syncer/base.rb +5 -8
- data/lib/backup/syncer/cloud/base.rb +27 -30
- data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
- data/lib/backup/syncer/cloud/local_file.rb +5 -8
- data/lib/backup/syncer/cloud/s3.rb +23 -24
- data/lib/backup/syncer/rsync/base.rb +6 -10
- data/lib/backup/syncer/rsync/local.rb +1 -5
- data/lib/backup/syncer/rsync/pull.rb +6 -10
- data/lib/backup/syncer/rsync/push.rb +18 -22
- data/lib/backup/template.rb +9 -14
- data/lib/backup/utilities.rb +78 -69
- data/lib/backup/version.rb +1 -3
- metadata +107 -677
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Compressor
|
5
3
|
class Base
|
@@ -18,18 +16,17 @@ module Backup
|
|
18
16
|
##
|
19
17
|
# Return the compressor name, with Backup namespace removed
|
20
18
|
def compressor_name
|
21
|
-
self.class.to_s.sub(
|
19
|
+
self.class.to_s.sub("Backup::", "")
|
22
20
|
end
|
23
21
|
|
24
22
|
##
|
25
23
|
# Logs a message to the console and log file to inform
|
26
24
|
# the client that Backup is using the compressor
|
27
25
|
def log!
|
28
|
-
Logger.info "Using #{
|
29
|
-
" Command: '#{
|
30
|
-
" Ext: '#{
|
26
|
+
Logger.info "Using #{compressor_name} for compression.\n" \
|
27
|
+
" Command: '#{@cmd}'\n" \
|
28
|
+
" Ext: '#{@ext}'"
|
31
29
|
end
|
32
|
-
|
33
30
|
end
|
34
31
|
end
|
35
32
|
end
|
@@ -1,9 +1,6 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Compressor
|
5
3
|
class Bzip2 < Base
|
6
|
-
|
7
4
|
##
|
8
5
|
# Specify the level of compression to use.
|
9
6
|
#
|
@@ -24,16 +21,15 @@ module Backup
|
|
24
21
|
|
25
22
|
instance_eval(&block) if block_given?
|
26
23
|
|
27
|
-
@cmd = "#{
|
28
|
-
@ext =
|
24
|
+
@cmd = "#{utility(:bzip2)}#{options}"
|
25
|
+
@ext = ".bz2"
|
29
26
|
end
|
30
27
|
|
31
28
|
private
|
32
29
|
|
33
30
|
def options
|
34
|
-
" -#{
|
31
|
+
" -#{@level}" if @level
|
35
32
|
end
|
36
|
-
|
37
33
|
end
|
38
34
|
end
|
39
35
|
end
|
@@ -1,9 +1,6 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Compressor
|
5
3
|
class Custom < Base
|
6
|
-
|
7
4
|
##
|
8
5
|
# Specify the system command to invoke a compressor,
|
9
6
|
# including any command-line arguments.
|
@@ -36,9 +33,9 @@ module Backup
|
|
36
33
|
# Return the command line using the full path.
|
37
34
|
# Ensures the command exists and is executable.
|
38
35
|
def set_cmd
|
39
|
-
parts = @command.to_s.split(
|
36
|
+
parts = @command.to_s.split(" ")
|
40
37
|
parts[0] = utility(parts[0])
|
41
|
-
parts.join(
|
38
|
+
parts.join(" ")
|
42
39
|
end
|
43
40
|
|
44
41
|
##
|
@@ -47,7 +44,6 @@ module Backup
|
|
47
44
|
def set_ext
|
48
45
|
@extension.to_s.strip
|
49
46
|
end
|
50
|
-
|
51
47
|
end
|
52
48
|
end
|
53
49
|
end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Compressor
|
5
3
|
class Gzip < Base
|
@@ -35,8 +33,8 @@ module Backup
|
|
35
33
|
# Determine if +--rsyncable+ is supported and cache the result.
|
36
34
|
def self.has_rsyncable?
|
37
35
|
return @has_rsyncable unless @has_rsyncable.nil?
|
38
|
-
cmd = "#{
|
39
|
-
@has_rsyncable =
|
36
|
+
cmd = "#{utility(:gzip)} --rsyncable --version >/dev/null 2>&1; echo $?"
|
37
|
+
@has_rsyncable = `#{cmd}`.chomp == "0"
|
40
38
|
end
|
41
39
|
|
42
40
|
##
|
@@ -49,26 +47,27 @@ module Backup
|
|
49
47
|
|
50
48
|
instance_eval(&block) if block_given?
|
51
49
|
|
52
|
-
@cmd = "#{
|
53
|
-
@ext =
|
50
|
+
@cmd = "#{utility(:gzip)}#{options}"
|
51
|
+
@ext = ".gz"
|
54
52
|
end
|
55
53
|
|
56
54
|
private
|
57
55
|
|
58
56
|
def options
|
59
|
-
opts =
|
60
|
-
opts << " -#{
|
61
|
-
if
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
57
|
+
opts = ""
|
58
|
+
opts << " -#{@level}" if @level
|
59
|
+
if @rsyncable
|
60
|
+
if self.class.has_rsyncable?
|
61
|
+
opts << " --rsyncable"
|
62
|
+
else
|
63
|
+
Logger.warn Error.new(<<-EOS)
|
64
|
+
'rsyncable' option ignored.
|
65
|
+
Your system's 'gzip' does not support the `--rsyncable` option.
|
66
|
+
EOS
|
67
|
+
end
|
68
|
+
end
|
69
69
|
opts
|
70
70
|
end
|
71
|
-
|
72
71
|
end
|
73
72
|
end
|
74
73
|
end
|
data/lib/backup/config.rb
CHANGED
@@ -1,15 +1,14 @@
|
|
1
|
-
|
2
|
-
require
|
3
|
-
require 'backup/config/helpers'
|
1
|
+
require "backup/config/dsl"
|
2
|
+
require "backup/config/helpers"
|
4
3
|
|
5
4
|
module Backup
|
6
5
|
module Config
|
7
6
|
class Error < Backup::Error; end
|
8
7
|
|
9
8
|
DEFAULTS = {
|
10
|
-
:
|
11
|
-
:
|
12
|
-
:
|
9
|
+
config_file: "config.rb",
|
10
|
+
data_path: ".data",
|
11
|
+
tmp_path: ".tmp"
|
13
12
|
}
|
14
13
|
|
15
14
|
class << self
|
@@ -19,20 +18,20 @@ module Backup
|
|
19
18
|
|
20
19
|
# Loads the user's +config.rb+ and all model files.
|
21
20
|
def load(options = {})
|
22
|
-
update(options)
|
21
|
+
update(options) # from the command line
|
23
22
|
|
24
23
|
unless File.exist?(config_file)
|
25
24
|
raise Error, "Could not find configuration file: '#{config_file}'."
|
26
25
|
end
|
27
26
|
|
28
27
|
config = File.read(config_file)
|
29
|
-
version = Backup::VERSION.split(
|
28
|
+
version = Backup::VERSION.split(".").first
|
30
29
|
unless config =~ /^# Backup v#{ version }\.x Configuration$/
|
31
30
|
raise Error, <<-EOS
|
32
31
|
Invalid Configuration File
|
33
|
-
The configuration file at '#{
|
34
|
-
does not appear to be a Backup v#{
|
35
|
-
If you have upgraded to v#{
|
32
|
+
The configuration file at '#{config_file}'
|
33
|
+
does not appear to be a Backup v#{version}.x configuration file.
|
34
|
+
If you have upgraded to v#{version}.x from a previous version,
|
36
35
|
you need to upgrade your configuration file.
|
37
36
|
Please see the instructions for upgrading in the Backup documentation.
|
38
37
|
EOS
|
@@ -44,7 +43,7 @@ module Backup
|
|
44
43
|
update(dsl._config_options) # from config.rb
|
45
44
|
update(options) # command line takes precedence
|
46
45
|
|
47
|
-
Dir[File.join(File.dirname(config_file),
|
46
|
+
Dir[File.join(File.dirname(config_file), "models", "*.rb")].each do |model|
|
48
47
|
dsl.instance_eval(File.read(model), model)
|
49
48
|
end
|
50
49
|
end
|
@@ -79,7 +78,7 @@ module Backup
|
|
79
78
|
raise Error, <<-EOS
|
80
79
|
Root Path Not Found
|
81
80
|
When specifying a --root-path, the path must exist.
|
82
|
-
Path was: #{
|
81
|
+
Path was: #{path}
|
83
82
|
EOS
|
84
83
|
end
|
85
84
|
@root_path = path
|
@@ -88,7 +87,7 @@ module Backup
|
|
88
87
|
def set_path_variable(name, path, ending, root_path)
|
89
88
|
# strip any trailing '/' in case the user supplied this as part of
|
90
89
|
# an absolute path, so we can match it against File.expand_path()
|
91
|
-
path = path.to_s.sub(/\/\s*$/,
|
90
|
+
path = path.to_s.sub(/\/\s*$/, "").lstrip
|
92
91
|
new_path = false
|
93
92
|
# If no path is given, the variable will not be set/updated
|
94
93
|
# unless a root_path was given. In which case the value will
|
@@ -108,12 +107,12 @@ module Backup
|
|
108
107
|
end
|
109
108
|
|
110
109
|
def reset!
|
111
|
-
@user = ENV[
|
112
|
-
@root_path = File.join(File.expand_path(ENV[
|
113
|
-
update(:
|
110
|
+
@user = ENV["USER"] || Etc.getpwuid.name
|
111
|
+
@root_path = File.join(File.expand_path(ENV["HOME"] || ""), "Backup")
|
112
|
+
update(root_path: @root_path)
|
114
113
|
end
|
115
114
|
end
|
116
115
|
|
117
|
-
reset!
|
116
|
+
reset! # set defaults on load
|
118
117
|
end
|
119
118
|
end
|
data/lib/backup/config/dsl.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Config
|
5
3
|
# Context for loading user config.rb and model files.
|
@@ -27,23 +25,23 @@ module Backup
|
|
27
25
|
create_modules(
|
28
26
|
DSL,
|
29
27
|
[ # Databases
|
30
|
-
[
|
28
|
+
["MySQL", "PostgreSQL", "MongoDB", "Redis", "Riak", "OpenLDAP", "SQLite"],
|
31
29
|
# Storages
|
32
|
-
[
|
33
|
-
|
30
|
+
["S3", "CloudFiles", "Dropbox", "FTP",
|
31
|
+
"SFTP", "SCP", "RSync", "Local", "Qiniu"],
|
34
32
|
# Compressors
|
35
|
-
[
|
33
|
+
["Gzip", "Bzip2", "Custom"],
|
36
34
|
# Encryptors
|
37
|
-
[
|
35
|
+
["OpenSSL", "GPG"],
|
38
36
|
# Syncers
|
39
37
|
[
|
40
|
-
{
|
41
|
-
{
|
38
|
+
{ "Cloud" => ["CloudFiles", "S3"] },
|
39
|
+
{ "RSync" => ["Push", "Pull", "Local"] }
|
42
40
|
],
|
43
41
|
# Notifiers
|
44
|
-
[
|
45
|
-
|
46
|
-
|
42
|
+
["Mail", "Twitter", "Campfire", "Prowl",
|
43
|
+
"Hipchat", "PagerDuty", "Pushover", "HttpPost", "Nagios",
|
44
|
+
"Slack", "FlowDock", "Zabbix", "Ses", "DataDog", "Command"]
|
47
45
|
]
|
48
46
|
)
|
49
47
|
end
|
@@ -69,7 +67,7 @@ module Backup
|
|
69
67
|
end
|
70
68
|
end
|
71
69
|
|
72
|
-
add_dsl_constants
|
70
|
+
add_dsl_constants # add constants on load
|
73
71
|
|
74
72
|
attr_reader :_config_options
|
75
73
|
|
@@ -79,25 +77,26 @@ module Backup
|
|
79
77
|
|
80
78
|
# Allow users to set command line path options in config.rb
|
81
79
|
[:root_path, :data_path, :tmp_path].each do |name|
|
82
|
-
define_method name
|
80
|
+
define_method name do |path|
|
81
|
+
_config_options[name] = path
|
82
|
+
end
|
83
83
|
end
|
84
84
|
|
85
85
|
# Allows users to create preconfigured models.
|
86
86
|
def preconfigure(name, &block)
|
87
87
|
unless name.is_a?(String) && name =~ /^[A-Z]/
|
88
|
-
raise Error, "Preconfigured model names must be given as a string "
|
88
|
+
raise Error, "Preconfigured model names must be given as a string " \
|
89
89
|
"and start with a capital letter."
|
90
90
|
end
|
91
91
|
|
92
92
|
if DSL.const_defined?(name)
|
93
|
-
raise Error, "'#{
|
93
|
+
raise Error, "'#{name}' is already in use " \
|
94
94
|
"and can not be used for a preconfigured model."
|
95
95
|
end
|
96
96
|
|
97
97
|
DSL.const_set(name, Class.new(Model))
|
98
98
|
DSL.const_get(name).preconfigure(&block)
|
99
99
|
end
|
100
|
-
|
101
100
|
end
|
102
101
|
end
|
103
102
|
end
|
@@ -1,16 +1,13 @@
|
|
1
|
-
|
2
|
-
require 'ostruct'
|
1
|
+
require "ostruct"
|
3
2
|
|
4
3
|
module Backup
|
5
4
|
module Config
|
6
5
|
module Helpers
|
7
|
-
|
8
6
|
def self.included(klass)
|
9
7
|
klass.extend ClassMethods
|
10
8
|
end
|
11
9
|
|
12
10
|
module ClassMethods
|
13
|
-
|
14
11
|
def defaults
|
15
12
|
@defaults ||= Config::Defaults.new
|
16
13
|
|
@@ -31,12 +28,12 @@ module Backup
|
|
31
28
|
end
|
32
29
|
|
33
30
|
def log_deprecation_warning(name, deprecation)
|
34
|
-
msg = "#{
|
35
|
-
"backup v.#{
|
36
|
-
msg << "\n#{
|
31
|
+
msg = "#{self}##{name} has been deprecated as of " \
|
32
|
+
"backup v.#{deprecation[:version]}"
|
33
|
+
msg << "\n#{deprecation[:message]}" if deprecation[:message]
|
37
34
|
Logger.warn Config::Error.new(<<-EOS)
|
38
35
|
[DEPRECATION WARNING]
|
39
|
-
#{
|
36
|
+
#{msg}
|
40
37
|
EOS
|
41
38
|
end
|
42
39
|
|
@@ -65,12 +62,11 @@ module Backup
|
|
65
62
|
#
|
66
63
|
def attr_deprecate(name, args = {})
|
67
64
|
deprecations[name] = {
|
68
|
-
:
|
69
|
-
:
|
70
|
-
:
|
65
|
+
version: nil,
|
66
|
+
message: nil,
|
67
|
+
action: nil
|
71
68
|
}.merge(args)
|
72
69
|
end
|
73
|
-
|
74
70
|
end # ClassMethods
|
75
71
|
|
76
72
|
private
|
@@ -107,10 +103,10 @@ module Backup
|
|
107
103
|
#
|
108
104
|
def method_missing(name, *args)
|
109
105
|
deprecation = nil
|
110
|
-
if method = name.to_s.chomp!(
|
106
|
+
if method = name.to_s.chomp!("=")
|
111
107
|
if (len = args.count) != 1
|
112
108
|
raise ArgumentError,
|
113
|
-
"wrong number of arguments (#{
|
109
|
+
"wrong number of arguments (#{len} for 1)", caller(1)
|
114
110
|
end
|
115
111
|
deprecation = self.class.deprecations[method.to_sym]
|
116
112
|
end
|
@@ -122,7 +118,6 @@ module Backup
|
|
122
118
|
super
|
123
119
|
end
|
124
120
|
end
|
125
|
-
|
126
121
|
end # Helpers
|
127
122
|
|
128
123
|
# Store for pre-configured defaults.
|
@@ -138,6 +133,5 @@ module Backup
|
|
138
133
|
@table.clear
|
139
134
|
end
|
140
135
|
end
|
141
|
-
|
142
136
|
end
|
143
137
|
end
|
data/lib/backup/database/base.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Database
|
5
3
|
class Error < Backup::Error; end
|
@@ -16,8 +14,8 @@ module Backup
|
|
16
14
|
# the model.
|
17
15
|
def initialize(model, database_id = nil)
|
18
16
|
@model = model
|
19
|
-
@database_id = database_id.to_s.gsub(/\W/,
|
20
|
-
@dump_path = File.join(Config.tmp_path, model.trigger,
|
17
|
+
@database_id = database_id.to_s.gsub(/\W/, "_") if database_id
|
18
|
+
@dump_path = File.join(Config.tmp_path, model.trigger, "databases")
|
21
19
|
load_defaults!
|
22
20
|
end
|
23
21
|
|
@@ -45,11 +43,13 @@ module Backup
|
|
45
43
|
# Model#initialize calls this method *after* all defined databases have
|
46
44
|
# been initialized so `backup check` can report these warnings.
|
47
45
|
def dump_filename
|
48
|
-
@dump_filename ||=
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
46
|
+
@dump_filename ||=
|
47
|
+
begin
|
48
|
+
unless database_id
|
49
|
+
if model.databases.select { |d| d.class == self.class }.count > 1
|
50
|
+
sleep 1
|
51
|
+
@database_id = Time.now.to_i.to_s[-5, 5]
|
52
|
+
Logger.warn Error.new(<<-EOS)
|
53
53
|
Database Identifier Missing
|
54
54
|
When multiple Databases are configured in a single Backup Model
|
55
55
|
that have the same class (MySQL, PostgreSQL, etc.), the optional
|
@@ -58,27 +58,28 @@ module Backup
|
|
58
58
|
This will result in an output file in your final backup package like:
|
59
59
|
databases/MySQL-database_id.sql
|
60
60
|
|
61
|
-
Backup has auto-generated an identifier (#{
|
61
|
+
Backup has auto-generated an identifier (#{database_id}) for this
|
62
62
|
database dump and will now continue.
|
63
|
-
|
63
|
+
EOS
|
64
|
+
end
|
64
65
|
end
|
65
|
-
end
|
66
66
|
|
67
|
-
|
68
|
-
|
67
|
+
self.class.name.split("::").last + (database_id ? "-#{database_id}" : "")
|
68
|
+
end
|
69
69
|
end
|
70
70
|
|
71
71
|
def database_name
|
72
|
-
@database_name ||= self.class.to_s.sub(
|
73
|
-
|
72
|
+
@database_name ||= self.class.to_s.sub("Backup::", "") +
|
73
|
+
(database_id ? " (#{database_id})" : "")
|
74
74
|
end
|
75
75
|
|
76
76
|
def log!(action)
|
77
|
-
msg =
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
77
|
+
msg =
|
78
|
+
case action
|
79
|
+
when :started then "Started..."
|
80
|
+
when :finished then "Finished!"
|
81
|
+
end
|
82
|
+
Logger.info "#{database_name} #{msg}"
|
82
83
|
end
|
83
84
|
end
|
84
85
|
end
|