backup 4.3.0 → 5.0.0.beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +5 -5
  2. data/LICENSE +19 -0
  3. data/README.md +13 -9
  4. data/bin/docker_test +24 -0
  5. data/lib/backup.rb +74 -78
  6. data/lib/backup/archive.rb +31 -32
  7. data/lib/backup/binder.rb +2 -6
  8. data/lib/backup/cleaner.rb +14 -18
  9. data/lib/backup/cli.rb +104 -108
  10. data/lib/backup/cloud_io/base.rb +4 -7
  11. data/lib/backup/cloud_io/cloud_files.rb +60 -62
  12. data/lib/backup/cloud_io/s3.rb +69 -76
  13. data/lib/backup/compressor/base.rb +4 -7
  14. data/lib/backup/compressor/bzip2.rb +3 -7
  15. data/lib/backup/compressor/custom.rb +2 -6
  16. data/lib/backup/compressor/gzip.rb +16 -17
  17. data/lib/backup/config.rb +17 -18
  18. data/lib/backup/config/dsl.rb +16 -17
  19. data/lib/backup/config/helpers.rb +10 -16
  20. data/lib/backup/database/base.rb +22 -21
  21. data/lib/backup/database/mongodb.rb +36 -37
  22. data/lib/backup/database/mysql.rb +40 -41
  23. data/lib/backup/database/openldap.rb +8 -10
  24. data/lib/backup/database/postgresql.rb +29 -30
  25. data/lib/backup/database/redis.rb +27 -30
  26. data/lib/backup/database/riak.rb +15 -18
  27. data/lib/backup/database/sqlite.rb +4 -6
  28. data/lib/backup/encryptor/base.rb +2 -4
  29. data/lib/backup/encryptor/gpg.rb +49 -59
  30. data/lib/backup/encryptor/open_ssl.rb +11 -14
  31. data/lib/backup/errors.rb +7 -12
  32. data/lib/backup/logger.rb +16 -18
  33. data/lib/backup/logger/console.rb +5 -8
  34. data/lib/backup/logger/fog_adapter.rb +2 -6
  35. data/lib/backup/logger/logfile.rb +10 -12
  36. data/lib/backup/logger/syslog.rb +2 -4
  37. data/lib/backup/model.rb +33 -40
  38. data/lib/backup/notifier/base.rb +24 -26
  39. data/lib/backup/notifier/campfire.rb +9 -11
  40. data/lib/backup/notifier/command.rb +3 -3
  41. data/lib/backup/notifier/datadog.rb +9 -12
  42. data/lib/backup/notifier/flowdock.rb +13 -17
  43. data/lib/backup/notifier/hipchat.rb +18 -14
  44. data/lib/backup/notifier/http_post.rb +11 -14
  45. data/lib/backup/notifier/mail.rb +42 -54
  46. data/lib/backup/notifier/nagios.rb +5 -9
  47. data/lib/backup/notifier/pagerduty.rb +10 -12
  48. data/lib/backup/notifier/prowl.rb +15 -15
  49. data/lib/backup/notifier/pushover.rb +7 -10
  50. data/lib/backup/notifier/ses.rb +52 -17
  51. data/lib/backup/notifier/slack.rb +39 -40
  52. data/lib/backup/notifier/twitter.rb +2 -5
  53. data/lib/backup/notifier/zabbix.rb +11 -14
  54. data/lib/backup/package.rb +5 -9
  55. data/lib/backup/packager.rb +16 -17
  56. data/lib/backup/pipeline.rb +17 -21
  57. data/lib/backup/splitter.rb +8 -11
  58. data/lib/backup/storage/base.rb +5 -8
  59. data/lib/backup/storage/cloud_files.rb +21 -23
  60. data/lib/backup/storage/cycler.rb +10 -15
  61. data/lib/backup/storage/dropbox.rb +15 -21
  62. data/lib/backup/storage/ftp.rb +14 -10
  63. data/lib/backup/storage/local.rb +5 -8
  64. data/lib/backup/storage/qiniu.rb +8 -8
  65. data/lib/backup/storage/rsync.rb +24 -26
  66. data/lib/backup/storage/s3.rb +27 -28
  67. data/lib/backup/storage/scp.rb +10 -12
  68. data/lib/backup/storage/sftp.rb +10 -12
  69. data/lib/backup/syncer/base.rb +5 -8
  70. data/lib/backup/syncer/cloud/base.rb +27 -30
  71. data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
  72. data/lib/backup/syncer/cloud/local_file.rb +5 -8
  73. data/lib/backup/syncer/cloud/s3.rb +23 -24
  74. data/lib/backup/syncer/rsync/base.rb +6 -10
  75. data/lib/backup/syncer/rsync/local.rb +1 -5
  76. data/lib/backup/syncer/rsync/pull.rb +6 -10
  77. data/lib/backup/syncer/rsync/push.rb +18 -22
  78. data/lib/backup/template.rb +9 -14
  79. data/lib/backup/utilities.rb +78 -69
  80. data/lib/backup/version.rb +1 -3
  81. metadata +107 -677
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Database
5
3
  class MongoDB < Base
@@ -60,7 +58,6 @@ module Backup
60
58
  lock_database if @lock
61
59
  dump!
62
60
  package!
63
-
64
61
  ensure
65
62
  unlock_database if @lock
66
63
  end
@@ -78,7 +75,7 @@ module Backup
78
75
  run(mongodump)
79
76
  else
80
77
  collections.each do |collection|
81
- run("#{ mongodump } --collection='#{ collection }'")
78
+ run("#{mongodump} --collection='#{collection}'")
82
79
  end
83
80
  end
84
81
  end
@@ -92,25 +89,27 @@ module Backup
92
89
  # If successful, +dump_packaging_path+ is removed.
93
90
  def package!
94
91
  pipeline = Pipeline.new
95
- dump_ext = 'tar'
92
+ dump_ext = "tar"
96
93
 
97
- pipeline << "#{ utility(:tar) } -cf - " +
98
- "-C '#{ dump_path }' '#{ dump_filename }'"
94
+ pipeline << "#{utility(:tar)} -cf - " \
95
+ "-C '#{dump_path}' '#{dump_filename}'"
99
96
 
100
- model.compressor.compress_with do |command, ext|
101
- pipeline << command
102
- dump_ext << ext
103
- end if model.compressor
97
+ if model.compressor
98
+ model.compressor.compress_with do |command, ext|
99
+ pipeline << command
100
+ dump_ext << ext
101
+ end
102
+ end
104
103
 
105
- pipeline << "#{ utility(:cat) } > " +
106
- "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
104
+ pipeline << "#{utility(:cat)} > " \
105
+ "'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
107
106
 
108
107
  pipeline.run
109
108
  if pipeline.success?
110
109
  FileUtils.rm_rf dump_packaging_path
111
110
  log!(:finished)
112
111
  else
113
- raise Error, "Dump Failed!\n" + pipeline.error_messages
112
+ raise Error, "Dump Failed!\n#{pipeline.error_messages}"
114
113
  end
115
114
  end
116
115
 
@@ -119,69 +118,69 @@ module Backup
119
118
  end
120
119
 
121
120
  def mongodump
122
- "#{ utility(:mongodump) } #{ name_option } #{ credential_options } " +
123
- "#{ connectivity_options } #{ ipv6_option } #{ oplog_option } " +
124
- "#{ user_options } --out='#{ dump_packaging_path }'"
121
+ "#{utility(:mongodump)} #{name_option} #{credential_options} " \
122
+ "#{connectivity_options} #{ipv6_option} #{oplog_option} " \
123
+ "#{user_options} --out='#{dump_packaging_path}'"
125
124
  end
126
125
 
127
126
  def name_option
128
- "--db='#{ name }'" if name
127
+ return unless name
128
+ "--db='#{name}'"
129
129
  end
130
130
 
131
131
  def credential_options
132
132
  opts = []
133
- opts << "--username='#{ username }'" if username
134
- opts << "--password='#{ password }'" if password
135
- opts << "--authenticationDatabase='#{ authdb }'" if authdb
136
- opts.join(' ')
133
+ opts << "--username='#{username}'" if username
134
+ opts << "--password='#{password}'" if password
135
+ opts << "--authenticationDatabase='#{authdb}'" if authdb
136
+ opts.join(" ")
137
137
  end
138
138
 
139
139
  def connectivity_options
140
140
  opts = []
141
- opts << "--host='#{ host }'" if host
142
- opts << "--port='#{ port }'" if port
143
- opts.join(' ')
141
+ opts << "--host='#{host}'" if host
142
+ opts << "--port='#{port}'" if port
143
+ opts.join(" ")
144
144
  end
145
145
 
146
146
  def ipv6_option
147
- '--ipv6' if ipv6
147
+ "--ipv6" if ipv6
148
148
  end
149
149
 
150
150
  def oplog_option
151
- '--oplog' if oplog
151
+ "--oplog" if oplog
152
152
  end
153
153
 
154
154
  def user_options
155
- Array(additional_options).join(' ')
155
+ Array(additional_options).join(" ")
156
156
  end
157
157
 
158
158
  def lock_database
159
- lock_command = <<-EOS.gsub(/^ +/, '')
159
+ lock_command = <<-EOS.gsub(/^ +/, "")
160
160
  echo 'use admin
161
161
  db.setProfilingLevel(0)
162
- db.fsyncLock()' | #{ mongo_shell }
162
+ db.fsyncLock()' | #{mongo_shell}
163
163
  EOS
164
164
 
165
165
  run(lock_command)
166
166
  end
167
167
 
168
168
  def unlock_database
169
- unlock_command = <<-EOS.gsub(/^ +/, '')
169
+ unlock_command = <<-EOS.gsub(/^ +/, "")
170
170
  echo 'use admin
171
- db.fsyncUnlock()' | #{ mongo_shell }
171
+ db.fsyncUnlock()' | #{mongo_shell}
172
172
  EOS
173
173
 
174
174
  run(unlock_command)
175
175
  end
176
176
 
177
177
  def mongo_shell
178
- cmd = "#{ utility(:mongo) } #{ connectivity_options }".rstrip
179
- cmd << " #{ credential_options }".rstrip
180
- cmd << " #{ ipv6_option }".rstrip
181
- cmd << " '#{ name }'" if name
178
+ cmd = "#{utility(:mongo)} #{connectivity_options}".rstrip
179
+ cmd << " #{credential_options}".rstrip
180
+ cmd << " #{ipv6_option}".rstrip
181
+ cmd << " '#{name}'" if name
182
182
  cmd
183
183
  end
184
-
185
184
  end
186
185
  end
187
186
  end
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Database
5
3
  class MySQL < Base
@@ -78,71 +76,73 @@ module Backup
78
76
  super
79
77
 
80
78
  pipeline = Pipeline.new
81
- dump_ext = sql_backup? ? 'sql' : 'tar'
79
+ dump_ext = sql_backup? ? "sql" : "tar"
82
80
 
83
81
  pipeline << sudo_option(sql_backup? ? mysqldump : innobackupex)
84
82
 
85
- model.compressor.compress_with do |command, ext|
86
- pipeline << command
87
- dump_ext << ext
88
- end if model.compressor
83
+ if model.compressor
84
+ model.compressor.compress_with do |command, ext|
85
+ pipeline << command
86
+ dump_ext << ext
87
+ end
88
+ end
89
89
 
90
- pipeline << "#{ utility(:cat) } > " +
91
- "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
90
+ pipeline << "#{utility(:cat)} > " \
91
+ "'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
92
92
 
93
93
  pipeline.run
94
94
  if pipeline.success?
95
95
  log!(:finished)
96
96
  else
97
- raise Error, "Dump Failed!\n" + pipeline.error_messages
97
+ raise Error, "Dump Failed!\n#{pipeline.error_messages}"
98
98
  end
99
99
  end
100
100
 
101
101
  private
102
102
 
103
103
  def mysqldump
104
- "#{ utility(:mysqldump) } #{ user_options } #{ credential_options } " +
105
- "#{ connectivity_options } #{ name_option } " +
106
- "#{ tables_to_dump } #{ tables_to_skip }"
104
+ "#{utility(:mysqldump)} #{user_options} #{credential_options} " \
105
+ "#{connectivity_options} #{name_option} " \
106
+ "#{tables_to_dump} #{tables_to_skip}"
107
107
  end
108
108
 
109
109
  def credential_options
110
110
  opts = []
111
- opts << "--user=#{ Shellwords.escape(username) }" if username
112
- opts << "--password=#{ Shellwords.escape(password) }" if password
113
- opts.join(' ')
111
+ opts << "--user=#{Shellwords.escape(username)}" if username
112
+ opts << "--password=#{Shellwords.escape(password)}" if password
113
+ opts.join(" ")
114
114
  end
115
115
 
116
116
  def connectivity_options
117
- return "--socket='#{ socket }'" if socket
117
+ return "--socket='#{socket}'" if socket
118
118
 
119
119
  opts = []
120
- opts << "--host='#{ host }'" if host
121
- opts << "--port='#{ port }'" if port
122
- opts.join(' ')
120
+ opts << "--host='#{host}'" if host
121
+ opts << "--port='#{port}'" if port
122
+ opts.join(" ")
123
123
  end
124
124
 
125
125
  def user_options
126
- Array(additional_options).join(' ')
126
+ Array(additional_options).join(" ")
127
127
  end
128
128
 
129
129
  def user_prepare_options
130
- Array(prepare_options).join(' ')
130
+ Array(prepare_options).join(" ")
131
131
  end
132
132
 
133
133
  def name_option
134
- dump_all? ? '--all-databases' : name
134
+ dump_all? ? "--all-databases" : name
135
135
  end
136
136
 
137
137
  def tables_to_dump
138
- Array(only_tables).join(' ') unless dump_all?
138
+ Array(only_tables).join(" ") unless dump_all?
139
139
  end
140
140
 
141
141
  def tables_to_skip
142
142
  Array(skip_tables).map do |table|
143
- table = (dump_all? || table['.']) ? table : "#{ name }.#{ table }"
144
- "--ignore-table='#{ table }'"
145
- end.join(' ')
143
+ table = dump_all? || table["."] ? table : "#{name}.#{table}"
144
+ "--ignore-table='#{table}'"
145
+ end.join(" ")
146
146
  end
147
147
 
148
148
  def dump_all?
@@ -155,28 +155,28 @@ module Backup
155
155
 
156
156
  def innobackupex
157
157
  # Creation phase
158
- "#{ utility(:innobackupex) } #{ credential_options } " +
159
- "#{ connectivity_options } #{ user_options } " +
160
- "--no-timestamp #{ temp_dir } #{ quiet_option } && " +
161
- innobackupex_prepare +
162
- # Move files to tar-ed stream on stdout
163
- "#{ utility(:tar) } --remove-files -cf - " +
164
- "-C #{ File.dirname(temp_dir) } #{ File.basename(temp_dir) }"
158
+ "#{utility(:innobackupex)} #{credential_options} " \
159
+ "#{connectivity_options} #{user_options} " \
160
+ "--no-timestamp #{temp_dir} #{quiet_option} && " +
161
+ innobackupex_prepare +
162
+ # Move files to tar-ed stream on stdout
163
+ "#{utility(:tar)} --remove-files -cf - " \
164
+ "-C #{File.dirname(temp_dir)} #{File.basename(temp_dir)}"
165
165
  end
166
166
 
167
167
  def innobackupex_prepare
168
168
  return "" unless @prepare_backup
169
169
  # Log applying phase (prepare for restore)
170
- "#{ utility(:innobackupex) } --apply-log #{ temp_dir } " +
171
- "#{ user_prepare_options } #{ quiet_option } && "
170
+ "#{utility(:innobackupex)} --apply-log #{temp_dir} " \
171
+ "#{user_prepare_options} #{quiet_option} && "
172
172
  end
173
173
 
174
174
  def sudo_option(command_block)
175
175
  return command_block unless sudo_user
176
176
 
177
- "sudo -s -u #{ sudo_user } -- <<END_OF_SUDO\n" +
178
- "#{command_block}\n" +
179
- "END_OF_SUDO\n"
177
+ "sudo -s -u #{sudo_user} -- <<END_OF_SUDO\n" \
178
+ "#{command_block}\n" \
179
+ "END_OF_SUDO\n"
180
180
  end
181
181
 
182
182
  def quiet_option
@@ -184,9 +184,8 @@ module Backup
184
184
  end
185
185
 
186
186
  def temp_dir
187
- File.join(dump_path, dump_filename + ".bkpdir")
187
+ File.join(dump_path, "#{dump_filename}.bkpdir")
188
188
  end
189
-
190
189
  end
191
190
  end
192
191
  end
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Database
5
3
  class OpenLDAP < Base
@@ -33,11 +31,11 @@ module Backup
33
31
  super
34
32
  instance_eval(&block) if block_given?
35
33
 
36
- @name ||= 'ldap_backup'
34
+ @name ||= "ldap_backup"
37
35
  @use_sudo ||= false
38
- @slapcat_args ||= Array.new
36
+ @slapcat_args ||= []
39
37
  @slapcat_utility ||= utility(:slapcat)
40
- @slapcat_conf ||= '/etc/ldap/slapd.d'
38
+ @slapcat_conf ||= "/etc/ldap/slapd.d"
41
39
  end
42
40
 
43
41
  ##
@@ -47,7 +45,7 @@ module Backup
47
45
  super
48
46
 
49
47
  pipeline = Pipeline.new
50
- dump_ext = 'ldif'
48
+ dump_ext = "ldif"
51
49
 
52
50
  pipeline << slapcat
53
51
  if @model.compressor
@@ -57,8 +55,8 @@ module Backup
57
55
  end
58
56
  end
59
57
 
60
- pipeline << "#{ utility(:cat) } > " +
61
- "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
58
+ pipeline << "#{utility(:cat)} > " \
59
+ "'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
62
60
 
63
61
  pipeline.run
64
62
  if pipeline.success?
@@ -73,7 +71,7 @@ module Backup
73
71
  ##
74
72
  # Builds the full slapcat string based on all attributes
75
73
  def slapcat
76
- command = "#{ slapcat_utility } #{ slapcat_conf_option } #{ slapcat_conf } #{ user_options }"
74
+ command = "#{slapcat_utility} #{slapcat_conf_option} #{slapcat_conf} #{user_options}"
77
75
  command.prepend("sudo ") if use_sudo
78
76
  command
79
77
  end
@@ -88,7 +86,7 @@ module Backup
88
86
  # Builds a compatible string for the additional options
89
87
  # specified by the user
90
88
  def user_options
91
- slapcat_args.join(' ')
89
+ slapcat_args.join(" ")
92
90
  end
93
91
  end
94
92
  end
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Database
5
3
  class PostgreSQL < Base
@@ -53,17 +51,19 @@ module Backup
53
51
  super
54
52
 
55
53
  pipeline = Pipeline.new
56
- dump_ext = 'sql'
54
+ dump_ext = "sql"
57
55
 
58
56
  pipeline << (dump_all? ? pgdumpall : pgdump)
59
57
 
60
- model.compressor.compress_with do |command, ext|
61
- pipeline << command
62
- dump_ext << ext
63
- end if model.compressor
58
+ if model.compressor
59
+ model.compressor.compress_with do |command, ext|
60
+ pipeline << command
61
+ dump_ext << ext
62
+ end
63
+ end
64
64
 
65
- pipeline << "#{ utility(:cat) } > " +
66
- "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
65
+ pipeline << "#{utility(:cat)} > " \
66
+ "'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
67
67
 
68
68
  pipeline.run
69
69
  if pipeline.success?
@@ -74,60 +74,59 @@ module Backup
74
74
  end
75
75
 
76
76
  def pgdump
77
- "#{ password_option }" +
78
- "#{ sudo_option }" +
79
- "#{ utility(:pg_dump) } #{ username_option } #{ connectivity_options } " +
80
- "#{ user_options } #{ tables_to_dump } #{ tables_to_skip } #{ name }"
77
+ password_option.to_s +
78
+ sudo_option.to_s +
79
+ "#{utility(:pg_dump)} #{username_option} #{connectivity_options} " \
80
+ "#{user_options} #{tables_to_dump} #{tables_to_skip} #{name}"
81
81
  end
82
82
 
83
83
  def pgdumpall
84
- "#{ password_option }" +
85
- "#{ sudo_option }" +
86
- "#{ utility(:pg_dumpall) } #{ username_option } " +
87
- "#{ connectivity_options } #{ user_options }"
84
+ password_option.to_s +
85
+ sudo_option.to_s +
86
+ "#{utility(:pg_dumpall)} #{username_option} " \
87
+ "#{connectivity_options} #{user_options}"
88
88
  end
89
89
 
90
90
  def password_option
91
- "PGPASSWORD=#{ Shellwords.escape(password) } " if password
91
+ "PGPASSWORD=#{Shellwords.escape(password)} " if password
92
92
  end
93
93
 
94
94
  def sudo_option
95
- "#{ utility(:sudo) } -n -u #{ sudo_user } " if sudo_user
95
+ "#{utility(:sudo)} -n -H -u #{sudo_user} " if sudo_user
96
96
  end
97
97
 
98
98
  def username_option
99
- "--username=#{ Shellwords.escape(username) }" if username
99
+ "--username=#{Shellwords.escape(username)}" if username
100
100
  end
101
101
 
102
102
  def connectivity_options
103
- return "--host='#{ socket }'" if socket
103
+ return "--host='#{socket}'" if socket
104
104
 
105
105
  opts = []
106
- opts << "--host='#{ host }'" if host
107
- opts << "--port='#{ port }'" if port
108
- opts.join(' ')
106
+ opts << "--host='#{host}'" if host
107
+ opts << "--port='#{port}'" if port
108
+ opts.join(" ")
109
109
  end
110
110
 
111
111
  def user_options
112
- Array(additional_options).join(' ')
112
+ Array(additional_options).join(" ")
113
113
  end
114
114
 
115
115
  def tables_to_dump
116
116
  Array(only_tables).map do |table|
117
- "--table='#{ table }'"
118
- end.join(' ')
117
+ "--table='#{table}'"
118
+ end.join(" ")
119
119
  end
120
120
 
121
121
  def tables_to_skip
122
122
  Array(skip_tables).map do |table|
123
- "--exclude-table='#{ table }'"
124
- end.join(' ')
123
+ "--exclude-table='#{table}'"
124
+ end.join(" ")
125
125
  end
126
126
 
127
127
  def dump_all?
128
128
  name == :all
129
129
  end
130
-
131
130
  end
132
131
  end
133
132
  end