backup 4.3.0 → 5.0.0.beta.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (81) hide show
  1. checksums.yaml +5 -5
  2. data/LICENSE +19 -0
  3. data/README.md +13 -9
  4. data/bin/docker_test +24 -0
  5. data/lib/backup.rb +74 -78
  6. data/lib/backup/archive.rb +31 -32
  7. data/lib/backup/binder.rb +2 -6
  8. data/lib/backup/cleaner.rb +14 -18
  9. data/lib/backup/cli.rb +104 -108
  10. data/lib/backup/cloud_io/base.rb +4 -7
  11. data/lib/backup/cloud_io/cloud_files.rb +60 -62
  12. data/lib/backup/cloud_io/s3.rb +69 -76
  13. data/lib/backup/compressor/base.rb +4 -7
  14. data/lib/backup/compressor/bzip2.rb +3 -7
  15. data/lib/backup/compressor/custom.rb +2 -6
  16. data/lib/backup/compressor/gzip.rb +16 -17
  17. data/lib/backup/config.rb +17 -18
  18. data/lib/backup/config/dsl.rb +16 -17
  19. data/lib/backup/config/helpers.rb +10 -16
  20. data/lib/backup/database/base.rb +22 -21
  21. data/lib/backup/database/mongodb.rb +36 -37
  22. data/lib/backup/database/mysql.rb +40 -41
  23. data/lib/backup/database/openldap.rb +8 -10
  24. data/lib/backup/database/postgresql.rb +29 -30
  25. data/lib/backup/database/redis.rb +27 -30
  26. data/lib/backup/database/riak.rb +15 -18
  27. data/lib/backup/database/sqlite.rb +4 -6
  28. data/lib/backup/encryptor/base.rb +2 -4
  29. data/lib/backup/encryptor/gpg.rb +49 -59
  30. data/lib/backup/encryptor/open_ssl.rb +11 -14
  31. data/lib/backup/errors.rb +7 -12
  32. data/lib/backup/logger.rb +16 -18
  33. data/lib/backup/logger/console.rb +5 -8
  34. data/lib/backup/logger/fog_adapter.rb +2 -6
  35. data/lib/backup/logger/logfile.rb +10 -12
  36. data/lib/backup/logger/syslog.rb +2 -4
  37. data/lib/backup/model.rb +33 -40
  38. data/lib/backup/notifier/base.rb +24 -26
  39. data/lib/backup/notifier/campfire.rb +9 -11
  40. data/lib/backup/notifier/command.rb +3 -3
  41. data/lib/backup/notifier/datadog.rb +9 -12
  42. data/lib/backup/notifier/flowdock.rb +13 -17
  43. data/lib/backup/notifier/hipchat.rb +18 -14
  44. data/lib/backup/notifier/http_post.rb +11 -14
  45. data/lib/backup/notifier/mail.rb +42 -54
  46. data/lib/backup/notifier/nagios.rb +5 -9
  47. data/lib/backup/notifier/pagerduty.rb +10 -12
  48. data/lib/backup/notifier/prowl.rb +15 -15
  49. data/lib/backup/notifier/pushover.rb +7 -10
  50. data/lib/backup/notifier/ses.rb +52 -17
  51. data/lib/backup/notifier/slack.rb +39 -40
  52. data/lib/backup/notifier/twitter.rb +2 -5
  53. data/lib/backup/notifier/zabbix.rb +11 -14
  54. data/lib/backup/package.rb +5 -9
  55. data/lib/backup/packager.rb +16 -17
  56. data/lib/backup/pipeline.rb +17 -21
  57. data/lib/backup/splitter.rb +8 -11
  58. data/lib/backup/storage/base.rb +5 -8
  59. data/lib/backup/storage/cloud_files.rb +21 -23
  60. data/lib/backup/storage/cycler.rb +10 -15
  61. data/lib/backup/storage/dropbox.rb +15 -21
  62. data/lib/backup/storage/ftp.rb +14 -10
  63. data/lib/backup/storage/local.rb +5 -8
  64. data/lib/backup/storage/qiniu.rb +8 -8
  65. data/lib/backup/storage/rsync.rb +24 -26
  66. data/lib/backup/storage/s3.rb +27 -28
  67. data/lib/backup/storage/scp.rb +10 -12
  68. data/lib/backup/storage/sftp.rb +10 -12
  69. data/lib/backup/syncer/base.rb +5 -8
  70. data/lib/backup/syncer/cloud/base.rb +27 -30
  71. data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
  72. data/lib/backup/syncer/cloud/local_file.rb +5 -8
  73. data/lib/backup/syncer/cloud/s3.rb +23 -24
  74. data/lib/backup/syncer/rsync/base.rb +6 -10
  75. data/lib/backup/syncer/rsync/local.rb +1 -5
  76. data/lib/backup/syncer/rsync/pull.rb +6 -10
  77. data/lib/backup/syncer/rsync/push.rb +18 -22
  78. data/lib/backup/template.rb +9 -14
  79. data/lib/backup/utilities.rb +78 -69
  80. data/lib/backup/version.rb +1 -3
  81. metadata +107 -677
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Database
5
3
  class MongoDB < Base
@@ -60,7 +58,6 @@ module Backup
60
58
  lock_database if @lock
61
59
  dump!
62
60
  package!
63
-
64
61
  ensure
65
62
  unlock_database if @lock
66
63
  end
@@ -78,7 +75,7 @@ module Backup
78
75
  run(mongodump)
79
76
  else
80
77
  collections.each do |collection|
81
- run("#{ mongodump } --collection='#{ collection }'")
78
+ run("#{mongodump} --collection='#{collection}'")
82
79
  end
83
80
  end
84
81
  end
@@ -92,25 +89,27 @@ module Backup
92
89
  # If successful, +dump_packaging_path+ is removed.
93
90
  def package!
94
91
  pipeline = Pipeline.new
95
- dump_ext = 'tar'
92
+ dump_ext = "tar"
96
93
 
97
- pipeline << "#{ utility(:tar) } -cf - " +
98
- "-C '#{ dump_path }' '#{ dump_filename }'"
94
+ pipeline << "#{utility(:tar)} -cf - " \
95
+ "-C '#{dump_path}' '#{dump_filename}'"
99
96
 
100
- model.compressor.compress_with do |command, ext|
101
- pipeline << command
102
- dump_ext << ext
103
- end if model.compressor
97
+ if model.compressor
98
+ model.compressor.compress_with do |command, ext|
99
+ pipeline << command
100
+ dump_ext << ext
101
+ end
102
+ end
104
103
 
105
- pipeline << "#{ utility(:cat) } > " +
106
- "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
104
+ pipeline << "#{utility(:cat)} > " \
105
+ "'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
107
106
 
108
107
  pipeline.run
109
108
  if pipeline.success?
110
109
  FileUtils.rm_rf dump_packaging_path
111
110
  log!(:finished)
112
111
  else
113
- raise Error, "Dump Failed!\n" + pipeline.error_messages
112
+ raise Error, "Dump Failed!\n#{pipeline.error_messages}"
114
113
  end
115
114
  end
116
115
 
@@ -119,69 +118,69 @@ module Backup
119
118
  end
120
119
 
121
120
  def mongodump
122
- "#{ utility(:mongodump) } #{ name_option } #{ credential_options } " +
123
- "#{ connectivity_options } #{ ipv6_option } #{ oplog_option } " +
124
- "#{ user_options } --out='#{ dump_packaging_path }'"
121
+ "#{utility(:mongodump)} #{name_option} #{credential_options} " \
122
+ "#{connectivity_options} #{ipv6_option} #{oplog_option} " \
123
+ "#{user_options} --out='#{dump_packaging_path}'"
125
124
  end
126
125
 
127
126
  def name_option
128
- "--db='#{ name }'" if name
127
+ return unless name
128
+ "--db='#{name}'"
129
129
  end
130
130
 
131
131
  def credential_options
132
132
  opts = []
133
- opts << "--username='#{ username }'" if username
134
- opts << "--password='#{ password }'" if password
135
- opts << "--authenticationDatabase='#{ authdb }'" if authdb
136
- opts.join(' ')
133
+ opts << "--username='#{username}'" if username
134
+ opts << "--password='#{password}'" if password
135
+ opts << "--authenticationDatabase='#{authdb}'" if authdb
136
+ opts.join(" ")
137
137
  end
138
138
 
139
139
  def connectivity_options
140
140
  opts = []
141
- opts << "--host='#{ host }'" if host
142
- opts << "--port='#{ port }'" if port
143
- opts.join(' ')
141
+ opts << "--host='#{host}'" if host
142
+ opts << "--port='#{port}'" if port
143
+ opts.join(" ")
144
144
  end
145
145
 
146
146
  def ipv6_option
147
- '--ipv6' if ipv6
147
+ "--ipv6" if ipv6
148
148
  end
149
149
 
150
150
  def oplog_option
151
- '--oplog' if oplog
151
+ "--oplog" if oplog
152
152
  end
153
153
 
154
154
  def user_options
155
- Array(additional_options).join(' ')
155
+ Array(additional_options).join(" ")
156
156
  end
157
157
 
158
158
  def lock_database
159
- lock_command = <<-EOS.gsub(/^ +/, '')
159
+ lock_command = <<-EOS.gsub(/^ +/, "")
160
160
  echo 'use admin
161
161
  db.setProfilingLevel(0)
162
- db.fsyncLock()' | #{ mongo_shell }
162
+ db.fsyncLock()' | #{mongo_shell}
163
163
  EOS
164
164
 
165
165
  run(lock_command)
166
166
  end
167
167
 
168
168
  def unlock_database
169
- unlock_command = <<-EOS.gsub(/^ +/, '')
169
+ unlock_command = <<-EOS.gsub(/^ +/, "")
170
170
  echo 'use admin
171
- db.fsyncUnlock()' | #{ mongo_shell }
171
+ db.fsyncUnlock()' | #{mongo_shell}
172
172
  EOS
173
173
 
174
174
  run(unlock_command)
175
175
  end
176
176
 
177
177
  def mongo_shell
178
- cmd = "#{ utility(:mongo) } #{ connectivity_options }".rstrip
179
- cmd << " #{ credential_options }".rstrip
180
- cmd << " #{ ipv6_option }".rstrip
181
- cmd << " '#{ name }'" if name
178
+ cmd = "#{utility(:mongo)} #{connectivity_options}".rstrip
179
+ cmd << " #{credential_options}".rstrip
180
+ cmd << " #{ipv6_option}".rstrip
181
+ cmd << " '#{name}'" if name
182
182
  cmd
183
183
  end
184
-
185
184
  end
186
185
  end
187
186
  end
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Database
5
3
  class MySQL < Base
@@ -78,71 +76,73 @@ module Backup
78
76
  super
79
77
 
80
78
  pipeline = Pipeline.new
81
- dump_ext = sql_backup? ? 'sql' : 'tar'
79
+ dump_ext = sql_backup? ? "sql" : "tar"
82
80
 
83
81
  pipeline << sudo_option(sql_backup? ? mysqldump : innobackupex)
84
82
 
85
- model.compressor.compress_with do |command, ext|
86
- pipeline << command
87
- dump_ext << ext
88
- end if model.compressor
83
+ if model.compressor
84
+ model.compressor.compress_with do |command, ext|
85
+ pipeline << command
86
+ dump_ext << ext
87
+ end
88
+ end
89
89
 
90
- pipeline << "#{ utility(:cat) } > " +
91
- "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
90
+ pipeline << "#{utility(:cat)} > " \
91
+ "'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
92
92
 
93
93
  pipeline.run
94
94
  if pipeline.success?
95
95
  log!(:finished)
96
96
  else
97
- raise Error, "Dump Failed!\n" + pipeline.error_messages
97
+ raise Error, "Dump Failed!\n#{pipeline.error_messages}"
98
98
  end
99
99
  end
100
100
 
101
101
  private
102
102
 
103
103
  def mysqldump
104
- "#{ utility(:mysqldump) } #{ user_options } #{ credential_options } " +
105
- "#{ connectivity_options } #{ name_option } " +
106
- "#{ tables_to_dump } #{ tables_to_skip }"
104
+ "#{utility(:mysqldump)} #{user_options} #{credential_options} " \
105
+ "#{connectivity_options} #{name_option} " \
106
+ "#{tables_to_dump} #{tables_to_skip}"
107
107
  end
108
108
 
109
109
  def credential_options
110
110
  opts = []
111
- opts << "--user=#{ Shellwords.escape(username) }" if username
112
- opts << "--password=#{ Shellwords.escape(password) }" if password
113
- opts.join(' ')
111
+ opts << "--user=#{Shellwords.escape(username)}" if username
112
+ opts << "--password=#{Shellwords.escape(password)}" if password
113
+ opts.join(" ")
114
114
  end
115
115
 
116
116
  def connectivity_options
117
- return "--socket='#{ socket }'" if socket
117
+ return "--socket='#{socket}'" if socket
118
118
 
119
119
  opts = []
120
- opts << "--host='#{ host }'" if host
121
- opts << "--port='#{ port }'" if port
122
- opts.join(' ')
120
+ opts << "--host='#{host}'" if host
121
+ opts << "--port='#{port}'" if port
122
+ opts.join(" ")
123
123
  end
124
124
 
125
125
  def user_options
126
- Array(additional_options).join(' ')
126
+ Array(additional_options).join(" ")
127
127
  end
128
128
 
129
129
  def user_prepare_options
130
- Array(prepare_options).join(' ')
130
+ Array(prepare_options).join(" ")
131
131
  end
132
132
 
133
133
  def name_option
134
- dump_all? ? '--all-databases' : name
134
+ dump_all? ? "--all-databases" : name
135
135
  end
136
136
 
137
137
  def tables_to_dump
138
- Array(only_tables).join(' ') unless dump_all?
138
+ Array(only_tables).join(" ") unless dump_all?
139
139
  end
140
140
 
141
141
  def tables_to_skip
142
142
  Array(skip_tables).map do |table|
143
- table = (dump_all? || table['.']) ? table : "#{ name }.#{ table }"
144
- "--ignore-table='#{ table }'"
145
- end.join(' ')
143
+ table = dump_all? || table["."] ? table : "#{name}.#{table}"
144
+ "--ignore-table='#{table}'"
145
+ end.join(" ")
146
146
  end
147
147
 
148
148
  def dump_all?
@@ -155,28 +155,28 @@ module Backup
155
155
 
156
156
  def innobackupex
157
157
  # Creation phase
158
- "#{ utility(:innobackupex) } #{ credential_options } " +
159
- "#{ connectivity_options } #{ user_options } " +
160
- "--no-timestamp #{ temp_dir } #{ quiet_option } && " +
161
- innobackupex_prepare +
162
- # Move files to tar-ed stream on stdout
163
- "#{ utility(:tar) } --remove-files -cf - " +
164
- "-C #{ File.dirname(temp_dir) } #{ File.basename(temp_dir) }"
158
+ "#{utility(:innobackupex)} #{credential_options} " \
159
+ "#{connectivity_options} #{user_options} " \
160
+ "--no-timestamp #{temp_dir} #{quiet_option} && " +
161
+ innobackupex_prepare +
162
+ # Move files to tar-ed stream on stdout
163
+ "#{utility(:tar)} --remove-files -cf - " \
164
+ "-C #{File.dirname(temp_dir)} #{File.basename(temp_dir)}"
165
165
  end
166
166
 
167
167
  def innobackupex_prepare
168
168
  return "" unless @prepare_backup
169
169
  # Log applying phase (prepare for restore)
170
- "#{ utility(:innobackupex) } --apply-log #{ temp_dir } " +
171
- "#{ user_prepare_options } #{ quiet_option } && "
170
+ "#{utility(:innobackupex)} --apply-log #{temp_dir} " \
171
+ "#{user_prepare_options} #{quiet_option} && "
172
172
  end
173
173
 
174
174
  def sudo_option(command_block)
175
175
  return command_block unless sudo_user
176
176
 
177
- "sudo -s -u #{ sudo_user } -- <<END_OF_SUDO\n" +
178
- "#{command_block}\n" +
179
- "END_OF_SUDO\n"
177
+ "sudo -s -u #{sudo_user} -- <<END_OF_SUDO\n" \
178
+ "#{command_block}\n" \
179
+ "END_OF_SUDO\n"
180
180
  end
181
181
 
182
182
  def quiet_option
@@ -184,9 +184,8 @@ module Backup
184
184
  end
185
185
 
186
186
  def temp_dir
187
- File.join(dump_path, dump_filename + ".bkpdir")
187
+ File.join(dump_path, "#{dump_filename}.bkpdir")
188
188
  end
189
-
190
189
  end
191
190
  end
192
191
  end
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Database
5
3
  class OpenLDAP < Base
@@ -33,11 +31,11 @@ module Backup
33
31
  super
34
32
  instance_eval(&block) if block_given?
35
33
 
36
- @name ||= 'ldap_backup'
34
+ @name ||= "ldap_backup"
37
35
  @use_sudo ||= false
38
- @slapcat_args ||= Array.new
36
+ @slapcat_args ||= []
39
37
  @slapcat_utility ||= utility(:slapcat)
40
- @slapcat_conf ||= '/etc/ldap/slapd.d'
38
+ @slapcat_conf ||= "/etc/ldap/slapd.d"
41
39
  end
42
40
 
43
41
  ##
@@ -47,7 +45,7 @@ module Backup
47
45
  super
48
46
 
49
47
  pipeline = Pipeline.new
50
- dump_ext = 'ldif'
48
+ dump_ext = "ldif"
51
49
 
52
50
  pipeline << slapcat
53
51
  if @model.compressor
@@ -57,8 +55,8 @@ module Backup
57
55
  end
58
56
  end
59
57
 
60
- pipeline << "#{ utility(:cat) } > " +
61
- "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
58
+ pipeline << "#{utility(:cat)} > " \
59
+ "'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
62
60
 
63
61
  pipeline.run
64
62
  if pipeline.success?
@@ -73,7 +71,7 @@ module Backup
73
71
  ##
74
72
  # Builds the full slapcat string based on all attributes
75
73
  def slapcat
76
- command = "#{ slapcat_utility } #{ slapcat_conf_option } #{ slapcat_conf } #{ user_options }"
74
+ command = "#{slapcat_utility} #{slapcat_conf_option} #{slapcat_conf} #{user_options}"
77
75
  command.prepend("sudo ") if use_sudo
78
76
  command
79
77
  end
@@ -88,7 +86,7 @@ module Backup
88
86
  # Builds a compatible string for the additional options
89
87
  # specified by the user
90
88
  def user_options
91
- slapcat_args.join(' ')
89
+ slapcat_args.join(" ")
92
90
  end
93
91
  end
94
92
  end
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Database
5
3
  class PostgreSQL < Base
@@ -53,17 +51,19 @@ module Backup
53
51
  super
54
52
 
55
53
  pipeline = Pipeline.new
56
- dump_ext = 'sql'
54
+ dump_ext = "sql"
57
55
 
58
56
  pipeline << (dump_all? ? pgdumpall : pgdump)
59
57
 
60
- model.compressor.compress_with do |command, ext|
61
- pipeline << command
62
- dump_ext << ext
63
- end if model.compressor
58
+ if model.compressor
59
+ model.compressor.compress_with do |command, ext|
60
+ pipeline << command
61
+ dump_ext << ext
62
+ end
63
+ end
64
64
 
65
- pipeline << "#{ utility(:cat) } > " +
66
- "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
65
+ pipeline << "#{utility(:cat)} > " \
66
+ "'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
67
67
 
68
68
  pipeline.run
69
69
  if pipeline.success?
@@ -74,60 +74,59 @@ module Backup
74
74
  end
75
75
 
76
76
  def pgdump
77
- "#{ password_option }" +
78
- "#{ sudo_option }" +
79
- "#{ utility(:pg_dump) } #{ username_option } #{ connectivity_options } " +
80
- "#{ user_options } #{ tables_to_dump } #{ tables_to_skip } #{ name }"
77
+ password_option.to_s +
78
+ sudo_option.to_s +
79
+ "#{utility(:pg_dump)} #{username_option} #{connectivity_options} " \
80
+ "#{user_options} #{tables_to_dump} #{tables_to_skip} #{name}"
81
81
  end
82
82
 
83
83
  def pgdumpall
84
- "#{ password_option }" +
85
- "#{ sudo_option }" +
86
- "#{ utility(:pg_dumpall) } #{ username_option } " +
87
- "#{ connectivity_options } #{ user_options }"
84
+ password_option.to_s +
85
+ sudo_option.to_s +
86
+ "#{utility(:pg_dumpall)} #{username_option} " \
87
+ "#{connectivity_options} #{user_options}"
88
88
  end
89
89
 
90
90
  def password_option
91
- "PGPASSWORD=#{ Shellwords.escape(password) } " if password
91
+ "PGPASSWORD=#{Shellwords.escape(password)} " if password
92
92
  end
93
93
 
94
94
  def sudo_option
95
- "#{ utility(:sudo) } -n -u #{ sudo_user } " if sudo_user
95
+ "#{utility(:sudo)} -n -H -u #{sudo_user} " if sudo_user
96
96
  end
97
97
 
98
98
  def username_option
99
- "--username=#{ Shellwords.escape(username) }" if username
99
+ "--username=#{Shellwords.escape(username)}" if username
100
100
  end
101
101
 
102
102
  def connectivity_options
103
- return "--host='#{ socket }'" if socket
103
+ return "--host='#{socket}'" if socket
104
104
 
105
105
  opts = []
106
- opts << "--host='#{ host }'" if host
107
- opts << "--port='#{ port }'" if port
108
- opts.join(' ')
106
+ opts << "--host='#{host}'" if host
107
+ opts << "--port='#{port}'" if port
108
+ opts.join(" ")
109
109
  end
110
110
 
111
111
  def user_options
112
- Array(additional_options).join(' ')
112
+ Array(additional_options).join(" ")
113
113
  end
114
114
 
115
115
  def tables_to_dump
116
116
  Array(only_tables).map do |table|
117
- "--table='#{ table }'"
118
- end.join(' ')
117
+ "--table='#{table}'"
118
+ end.join(" ")
119
119
  end
120
120
 
121
121
  def tables_to_skip
122
122
  Array(skip_tables).map do |table|
123
- "--exclude-table='#{ table }'"
124
- end.join(' ')
123
+ "--exclude-table='#{table}'"
124
+ end.join(" ")
125
125
  end
126
126
 
127
127
  def dump_all?
128
128
  name == :all
129
129
  end
130
-
131
130
  end
132
131
  end
133
132
  end