drebs 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +21 -0
- data/Rakefile +389 -0
- data/bin/drebs +249 -0
- data/drebs.gemspec +43 -0
- data/lib/drebs.rb +63 -0
- metadata +105 -0
data/README.md
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
# DREBS (Disaster Recovery for Elastic Block Store)
|
2
|
+
|
3
|
+
## About
|
4
|
+
* DREBS is a tool for taking periodic snapshots of EBS volumes.
|
5
|
+
* DREBS is designed to be run on the EC2 host which the EBS volumes to be snapshoted are attached.
|
6
|
+
* DREBS supports configurable retention strategies.
|
7
|
+
* DREBS supports configurable pre and post snapshot tasks such as dumping databases prior to snapshot for consistency.
|
8
|
+
|
9
|
+
## Installation & Setup
|
10
|
+
1. Clone the repo or install the gem
|
11
|
+
1. Currently configuration is located at the top of the drebs bin. Add you ec2 key, etc.
|
12
|
+
1. Add Crontab entry: 0 * * * * drebs
|
13
|
+
|
14
|
+
## Issues
|
15
|
+
* State including config is cached in drebs_state.json. This file will need to be deleted to get drebs to pick up new config. Doing so will orphan current snapshots which will need to be deleted manually.
|
16
|
+
|
17
|
+
## Todo
|
18
|
+
* Tests!
|
19
|
+
* Refactor using main with db for state and external config
|
20
|
+
* Use Whenever gem for crontab setup
|
21
|
+
* Arbitrary execution intervals (Snapshots every 5 minutes instead of every hour)
|
data/Rakefile
ADDED
@@ -0,0 +1,389 @@
|
|
1
|
+
This.rubyforge_project = 'DREBS'
|
2
|
+
This.author = "Garett Shulman"
|
3
|
+
This.email = "garett@dojo4.com"
|
4
|
+
This.homepage = "https://github.com/dojo4/#{ This.lib }"
|
5
|
+
|
6
|
+
|
7
|
+
task :default do
|
8
|
+
puts((Rake::Task.tasks.map{|task| task.name.gsub(/::/,':')} - ['default']).sort)
|
9
|
+
end
|
10
|
+
|
11
|
+
task :test do
|
12
|
+
run_tests!
|
13
|
+
end
|
14
|
+
|
15
|
+
namespace :test do
|
16
|
+
task(:unit){ run_tests!(:unit) }
|
17
|
+
task(:functional){ run_tests!(:functional) }
|
18
|
+
task(:integration){ run_tests!(:integration) }
|
19
|
+
end
|
20
|
+
|
21
|
+
def run_tests!(which = nil)
|
22
|
+
which ||= '**'
|
23
|
+
test_dir = File.join(This.dir, "test")
|
24
|
+
test_glob ||= File.join(test_dir, "#{ which }/**_test.rb")
|
25
|
+
test_rbs = Dir.glob(test_glob).sort
|
26
|
+
|
27
|
+
div = ('=' * 119)
|
28
|
+
line = ('-' * 119)
|
29
|
+
|
30
|
+
test_rbs.each_with_index do |test_rb, index|
|
31
|
+
testno = index + 1
|
32
|
+
command = "#{ File.basename(This.ruby) } -I ./lib -I ./test/lib #{ test_rb }"
|
33
|
+
|
34
|
+
puts
|
35
|
+
say(div, :color => :cyan, :bold => true)
|
36
|
+
say("@#{ testno } => ", :bold => true, :method => :print)
|
37
|
+
say(command, :color => :cyan, :bold => true)
|
38
|
+
say(line, :color => :cyan, :bold => true)
|
39
|
+
|
40
|
+
system(command)
|
41
|
+
|
42
|
+
say(line, :color => :cyan, :bold => true)
|
43
|
+
|
44
|
+
status = $?.exitstatus
|
45
|
+
|
46
|
+
if status.zero?
|
47
|
+
say("@#{ testno } <= ", :bold => true, :color => :white, :method => :print)
|
48
|
+
say("SUCCESS", :color => :green, :bold => true)
|
49
|
+
else
|
50
|
+
say("@#{ testno } <= ", :bold => true, :color => :white, :method => :print)
|
51
|
+
say("FAILURE", :color => :red, :bold => true)
|
52
|
+
end
|
53
|
+
say(line, :color => :cyan, :bold => true)
|
54
|
+
|
55
|
+
exit(status) unless status.zero?
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
|
60
|
+
task :gemspec do
|
61
|
+
ignore_extensions = ['git', 'svn', 'tmp', /sw./, 'bak', 'gem']
|
62
|
+
ignore_directories = ['pkg', 'db']
|
63
|
+
ignore_files = ['test/log', 'test/db.yml', 'a.rb', 'b.rb'] + Dir['db/*'] + %w'db'
|
64
|
+
|
65
|
+
shiteless =
|
66
|
+
lambda do |list|
|
67
|
+
list.delete_if do |entry|
|
68
|
+
next unless test(?e, entry)
|
69
|
+
extension = File.basename(entry).split(%r/[.]/).last
|
70
|
+
ignore_extensions.any?{|ext| ext === extension}
|
71
|
+
end
|
72
|
+
list.delete_if do |entry|
|
73
|
+
next unless test(?d, entry)
|
74
|
+
dirname = File.expand_path(entry)
|
75
|
+
ignore_directories.any?{|dir| File.expand_path(dir) == dirname}
|
76
|
+
end
|
77
|
+
list.delete_if do |entry|
|
78
|
+
next unless test(?f, entry)
|
79
|
+
filename = File.expand_path(entry)
|
80
|
+
ignore_files.any?{|file| File.expand_path(file) == filename}
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
lib = This.lib
|
85
|
+
object = This.object
|
86
|
+
version = This.version
|
87
|
+
files = shiteless[Dir::glob("**/**")]
|
88
|
+
executables = shiteless[Dir::glob("bin/*")].map{|exe| File.basename(exe)}
|
89
|
+
#has_rdoc = true #File.exist?('doc')
|
90
|
+
test_files = test(?e, "test/#{ lib }.rb") ? "test/#{ lib }.rb" : nil
|
91
|
+
summary = object.respond_to?(:summary) ? object.summary : "summary: #{ lib } kicks the ass"
|
92
|
+
description = object.respond_to?(:description) ? object.description : "description: #{ lib } kicks the ass"
|
93
|
+
|
94
|
+
if This.extensions.nil?
|
95
|
+
This.extensions = []
|
96
|
+
extensions = This.extensions
|
97
|
+
%w( Makefile configure extconf.rb ).each do |ext|
|
98
|
+
extensions << ext if File.exists?(ext)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
extensions = [extensions].flatten.compact
|
102
|
+
|
103
|
+
# TODO
|
104
|
+
if This.dependencies.nil?
|
105
|
+
dependencies = []
|
106
|
+
else
|
107
|
+
case This.dependencies
|
108
|
+
when Hash
|
109
|
+
dependencies = This.dependencies.values
|
110
|
+
when Array
|
111
|
+
dependencies = This.dependencies
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
template =
|
116
|
+
if test(?e, 'gemspec.erb')
|
117
|
+
Template{ IO.read('gemspec.erb') }
|
118
|
+
else
|
119
|
+
Template {
|
120
|
+
<<-__
|
121
|
+
## <%= lib %>.gemspec
|
122
|
+
#
|
123
|
+
|
124
|
+
Gem::Specification::new do |spec|
|
125
|
+
spec.name = <%= lib.inspect %>
|
126
|
+
spec.version = <%= version.inspect %>
|
127
|
+
spec.platform = Gem::Platform::RUBY
|
128
|
+
spec.summary = <%= lib.inspect %>
|
129
|
+
spec.description = <%= description.inspect %>
|
130
|
+
|
131
|
+
spec.files =\n<%= files.sort.pretty_inspect %>
|
132
|
+
spec.executables = <%= executables.inspect %>
|
133
|
+
spec.require_path = "lib"
|
134
|
+
|
135
|
+
spec.test_files = <%= test_files.inspect %>
|
136
|
+
|
137
|
+
<% dependencies.each do |lib_version| %>
|
138
|
+
spec.add_dependency(*<%= Array(lib_version).flatten.inspect %>)
|
139
|
+
<% end %>
|
140
|
+
|
141
|
+
spec.extensions.push(*<%= extensions.inspect %>)
|
142
|
+
|
143
|
+
spec.rubyforge_project = <%= This.rubyforge_project.inspect %>
|
144
|
+
spec.author = <%= This.author.inspect %>
|
145
|
+
spec.email = <%= This.email.inspect %>
|
146
|
+
spec.homepage = <%= This.homepage.inspect %>
|
147
|
+
end
|
148
|
+
__
|
149
|
+
}
|
150
|
+
end
|
151
|
+
|
152
|
+
Fu.mkdir_p(This.pkgdir)
|
153
|
+
gemspec = "#{ lib }.gemspec"
|
154
|
+
open(gemspec, "w"){|fd| fd.puts(template)}
|
155
|
+
This.gemspec = gemspec
|
156
|
+
end
|
157
|
+
|
158
|
+
task :gem => [:clean, :gemspec] do
|
159
|
+
Fu.mkdir_p(This.pkgdir)
|
160
|
+
before = Dir['*.gem']
|
161
|
+
cmd = "gem build #{ This.gemspec }"
|
162
|
+
`#{ cmd }`
|
163
|
+
after = Dir['*.gem']
|
164
|
+
gem = ((after - before).first || after.first) or abort('no gem!')
|
165
|
+
Fu.mv(gem, This.pkgdir)
|
166
|
+
This.gem = File.join(This.pkgdir, File.basename(gem))
|
167
|
+
end
|
168
|
+
|
169
|
+
task :readme do
|
170
|
+
samples = ''
|
171
|
+
prompt = '~ > '
|
172
|
+
lib = This.lib
|
173
|
+
version = This.version
|
174
|
+
|
175
|
+
Dir['sample*/*'].sort.each do |sample|
|
176
|
+
samples << "\n" << " <========< #{ sample } >========>" << "\n\n"
|
177
|
+
|
178
|
+
cmd = "cat #{ sample }"
|
179
|
+
samples << Util.indent(prompt + cmd, 2) << "\n\n"
|
180
|
+
samples << Util.indent(`#{ cmd }`, 4) << "\n"
|
181
|
+
|
182
|
+
cmd = "ruby #{ sample }"
|
183
|
+
samples << Util.indent(prompt + cmd, 2) << "\n\n"
|
184
|
+
|
185
|
+
cmd = "ruby -e'STDOUT.sync=true; exec %(ruby -I ./lib #{ sample })'"
|
186
|
+
samples << Util.indent(`#{ cmd } 2>&1`, 4) << "\n"
|
187
|
+
end
|
188
|
+
|
189
|
+
template =
|
190
|
+
if test(?e, 'readme.erb')
|
191
|
+
Template{ IO.read('readme.erb') }
|
192
|
+
else
|
193
|
+
Template {
|
194
|
+
<<-__
|
195
|
+
NAME
|
196
|
+
#{ lib }
|
197
|
+
|
198
|
+
DESCRIPTION
|
199
|
+
|
200
|
+
INSTALL
|
201
|
+
gem install #{ lib }
|
202
|
+
|
203
|
+
SAMPLES
|
204
|
+
#{ samples }
|
205
|
+
__
|
206
|
+
}
|
207
|
+
end
|
208
|
+
|
209
|
+
open("README", "w"){|fd| fd.puts template}
|
210
|
+
end
|
211
|
+
|
212
|
+
|
213
|
+
task :clean do
|
214
|
+
Dir[File.join(This.pkgdir, '**/**')].each{|entry| Fu.rm_rf(entry)}
|
215
|
+
end
|
216
|
+
|
217
|
+
|
218
|
+
task :release => [:clean, :gemspec, :gem] do
|
219
|
+
gems = Dir[File.join(This.pkgdir, '*.gem')].flatten
|
220
|
+
raise "which one? : #{ gems.inspect }" if gems.size > 1
|
221
|
+
raise "no gems?" if gems.size < 1
|
222
|
+
|
223
|
+
cmd = "gem push #{ This.gem }"
|
224
|
+
puts cmd
|
225
|
+
puts
|
226
|
+
system(cmd)
|
227
|
+
abort("cmd(#{ cmd }) failed with (#{ $?.inspect })") unless $?.exitstatus.zero?
|
228
|
+
|
229
|
+
cmd = "rubyforge login && rubyforge add_release #{ This.rubyforge_project } #{ This.lib } #{ This.version } #{ This.gem }"
|
230
|
+
puts cmd
|
231
|
+
puts
|
232
|
+
system(cmd)
|
233
|
+
abort("cmd(#{ cmd }) failed with (#{ $?.inspect })") unless $?.exitstatus.zero?
|
234
|
+
end
|
235
|
+
|
236
|
+
|
237
|
+
|
238
|
+
|
239
|
+
|
240
|
+
BEGIN {
|
241
|
+
# support for this rakefile
|
242
|
+
#
|
243
|
+
$VERBOSE = nil
|
244
|
+
|
245
|
+
require 'ostruct'
|
246
|
+
require 'erb'
|
247
|
+
require 'fileutils'
|
248
|
+
require 'rbconfig'
|
249
|
+
require 'pp'
|
250
|
+
|
251
|
+
# fu shortcut
|
252
|
+
#
|
253
|
+
Fu = FileUtils
|
254
|
+
|
255
|
+
# cache a bunch of stuff about this rakefile/environment
|
256
|
+
#
|
257
|
+
This = OpenStruct.new
|
258
|
+
|
259
|
+
This.file = File.expand_path(__FILE__)
|
260
|
+
This.dir = File.dirname(This.file)
|
261
|
+
This.pkgdir = File.join(This.dir, 'pkg')
|
262
|
+
|
263
|
+
# grok lib
|
264
|
+
#
|
265
|
+
lib = ENV['LIB']
|
266
|
+
unless lib
|
267
|
+
lib = File.basename(Dir.pwd).sub(/[-].*$/, '')
|
268
|
+
end
|
269
|
+
This.lib = lib
|
270
|
+
|
271
|
+
# grok version
|
272
|
+
#
|
273
|
+
version = ENV['VERSION']
|
274
|
+
unless version
|
275
|
+
require "./lib/#{ This.lib }"
|
276
|
+
This.name = lib.capitalize
|
277
|
+
This.object = eval(This.name)
|
278
|
+
version = This.object.send(:version)
|
279
|
+
end
|
280
|
+
This.version = version
|
281
|
+
|
282
|
+
# see if dependencies are export by the module
|
283
|
+
#
|
284
|
+
if This.object.respond_to?(:dependencies)
|
285
|
+
This.dependencies = This.object.dependencies
|
286
|
+
end
|
287
|
+
|
288
|
+
# we need to know the name of the lib an it's version
|
289
|
+
#
|
290
|
+
abort('no lib') unless This.lib
|
291
|
+
abort('no version') unless This.version
|
292
|
+
|
293
|
+
# discover full path to this ruby executable
|
294
|
+
#
|
295
|
+
c = Config::CONFIG
|
296
|
+
bindir = c["bindir"] || c['BINDIR']
|
297
|
+
ruby_install_name = c['ruby_install_name'] || c['RUBY_INSTALL_NAME'] || 'ruby'
|
298
|
+
ruby_ext = c['EXEEXT'] || ''
|
299
|
+
ruby = File.join(bindir, (ruby_install_name + ruby_ext))
|
300
|
+
This.ruby = ruby
|
301
|
+
|
302
|
+
# some utils
|
303
|
+
#
|
304
|
+
module Util
|
305
|
+
def indent(s, n = 2)
|
306
|
+
s = unindent(s)
|
307
|
+
ws = ' ' * n
|
308
|
+
s.gsub(%r/^/, ws)
|
309
|
+
end
|
310
|
+
|
311
|
+
def unindent(s)
|
312
|
+
indent = nil
|
313
|
+
s.each_line do |line|
|
314
|
+
next if line =~ %r/^\s*$/
|
315
|
+
indent = line[%r/^\s*/] and break
|
316
|
+
end
|
317
|
+
indent ? s.gsub(%r/^#{ indent }/, "") : s
|
318
|
+
end
|
319
|
+
extend self
|
320
|
+
end
|
321
|
+
|
322
|
+
# template support
|
323
|
+
#
|
324
|
+
class Template
|
325
|
+
def initialize(&block)
|
326
|
+
@block = block
|
327
|
+
@template = block.call.to_s
|
328
|
+
end
|
329
|
+
def expand(b=nil)
|
330
|
+
ERB.new(Util.unindent(@template)).result((b||@block).binding)
|
331
|
+
end
|
332
|
+
alias_method 'to_s', 'expand'
|
333
|
+
end
|
334
|
+
def Template(*args, &block) Template.new(*args, &block) end
|
335
|
+
|
336
|
+
# colored console output support
|
337
|
+
#
|
338
|
+
This.ansi = {
|
339
|
+
:clear => "\e[0m",
|
340
|
+
:reset => "\e[0m",
|
341
|
+
:erase_line => "\e[K",
|
342
|
+
:erase_char => "\e[P",
|
343
|
+
:bold => "\e[1m",
|
344
|
+
:dark => "\e[2m",
|
345
|
+
:underline => "\e[4m",
|
346
|
+
:underscore => "\e[4m",
|
347
|
+
:blink => "\e[5m",
|
348
|
+
:reverse => "\e[7m",
|
349
|
+
:concealed => "\e[8m",
|
350
|
+
:black => "\e[30m",
|
351
|
+
:red => "\e[31m",
|
352
|
+
:green => "\e[32m",
|
353
|
+
:yellow => "\e[33m",
|
354
|
+
:blue => "\e[34m",
|
355
|
+
:magenta => "\e[35m",
|
356
|
+
:cyan => "\e[36m",
|
357
|
+
:white => "\e[37m",
|
358
|
+
:on_black => "\e[40m",
|
359
|
+
:on_red => "\e[41m",
|
360
|
+
:on_green => "\e[42m",
|
361
|
+
:on_yellow => "\e[43m",
|
362
|
+
:on_blue => "\e[44m",
|
363
|
+
:on_magenta => "\e[45m",
|
364
|
+
:on_cyan => "\e[46m",
|
365
|
+
:on_white => "\e[47m"
|
366
|
+
}
|
367
|
+
def say(phrase, *args)
|
368
|
+
options = args.last.is_a?(Hash) ? args.pop : {}
|
369
|
+
options[:color] = args.shift.to_s.to_sym unless args.empty?
|
370
|
+
keys = options.keys
|
371
|
+
keys.each{|key| options[key.to_s.to_sym] = options.delete(key)}
|
372
|
+
|
373
|
+
color = options[:color]
|
374
|
+
bold = options.has_key?(:bold)
|
375
|
+
|
376
|
+
parts = [phrase]
|
377
|
+
parts.unshift(This.ansi[color]) if color
|
378
|
+
parts.unshift(This.ansi[:bold]) if bold
|
379
|
+
parts.push(This.ansi[:clear]) if parts.size > 1
|
380
|
+
|
381
|
+
method = options[:method] || :puts
|
382
|
+
|
383
|
+
Kernel.send(method, parts.join)
|
384
|
+
end
|
385
|
+
|
386
|
+
# always run out of the project dir
|
387
|
+
#
|
388
|
+
Dir.chdir(This.dir)
|
389
|
+
}
|
data/bin/drebs
ADDED
@@ -0,0 +1,249 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'right_aws'
|
5
|
+
require 'logger'
|
6
|
+
require 'main'
|
7
|
+
require 'systemu'
|
8
|
+
require 'json'
|
9
|
+
require 'socket'
|
10
|
+
require 'net/smtp'
|
11
|
+
|
12
|
+
class DREBS
|
13
|
+
|
14
|
+
###### Begin User Config
|
15
|
+
DREBS_HOST_NAME = 'An identifier for your host: www1'
|
16
|
+
AWS_ACCESS_KEY_ID = 'YOUR ACCESS KEY ID'
|
17
|
+
AWS_SECRET_ACCESS_KEY = 'YOUR SECRET ACCESS KEY'
|
18
|
+
REGION = 'us-west-1'
|
19
|
+
BACKUP_STRATEGY = [
|
20
|
+
{
|
21
|
+
'hours_between'=>1, 'num_to_keep'=>5,
|
22
|
+
'mount_point'=>'/dev/sdh',
|
23
|
+
'pre_snapshot_tasks'=> [
|
24
|
+
'pg_dump some_app_production > /path/to/backups/on/snapshoted/volume/some_app_production.sql',
|
25
|
+
'mongodump -d another_app-production -o /path/to/backups/on/snapshoted/volume/'
|
26
|
+
]
|
27
|
+
},
|
28
|
+
{
|
29
|
+
'hours_between'=>6, 'num_to_keep'=>4,
|
30
|
+
'mount_point'=>'/dev/sdh',
|
31
|
+
'pre_snapshot_tasks'=> []
|
32
|
+
},
|
33
|
+
{
|
34
|
+
'hours_between'=>24, 'num_to_keep'=>4,
|
35
|
+
'mount_point'=>'/dev/sda1',
|
36
|
+
'pre_snapshot_tasks'=> []
|
37
|
+
},
|
38
|
+
{
|
39
|
+
'hours_between'=>96, 'num_to_keep'=>4,
|
40
|
+
'mount_point'=>'/dev/sda1',
|
41
|
+
'pre_snapshot_tasks'=> []
|
42
|
+
}
|
43
|
+
]
|
44
|
+
LOG_PATH = '/usr/local/var/drebs.log'
|
45
|
+
BACKUP_STATE_FILE_PATH = '/usr/local/var/drebs_state.json'
|
46
|
+
EMAIL_ON_EXCEPTION = 'admin@your.org'
|
47
|
+
EMAIL_HOST = 'imap.gmail.com'
|
48
|
+
EMAIL_PORT = 993
|
49
|
+
EMAIL_USERNAME = 'your smpt username'
|
50
|
+
EMAIL_PASSWORD = 'your smpt password'
|
51
|
+
###### End User Config
|
52
|
+
|
53
|
+
def initialize(options = {})
|
54
|
+
@drebs_host_name = options[:drebs_host_name] || options['drebs_host_name'] || DREBS_HOST_NAME
|
55
|
+
@aws_access_key_id = options[:aws_access_key_id] || options['aws_access_key_id'] || AWS_ACCESS_KEY_ID
|
56
|
+
@aws_secret_access_key = options[:aws_secret_access_key] || options['aws_secret_access_key'] || AWS_SECRET_ACCESS_KEY
|
57
|
+
@region = options[:region] || options['region'] || REGION
|
58
|
+
@backup_strategy = options[:backup_strategy] || options['backup_strategy'] || BACKUP_STRATEGY
|
59
|
+
@log_path = options[:log_path] || options['log_path'] || LOG_PATH
|
60
|
+
@backup_state_file_path = options[:backup_state_file_path] || options['backup_state_file_path'] || BACKUP_STATE_FILE_PATH
|
61
|
+
@email_on_exception = options[:email_on_exception] || options['email_on_exception'] || EMAIL_ON_EXCEPTION
|
62
|
+
@log = Logger.new(@log_path, 0, 10 * 1024 * 1024)
|
63
|
+
end
|
64
|
+
|
65
|
+
def setup_backup_data(backup_strategy=@backup_strategy)
|
66
|
+
backup_data = Marshal.load(Marshal.dump(@backup_strategy))
|
67
|
+
backup_data.collect{|a_backup_strategy|
|
68
|
+
a_backup_strategy['hours_until_next_run'] = a_backup_strategy['hours_between']
|
69
|
+
a_backup_strategy['previous_snapshots'] = []
|
70
|
+
}
|
71
|
+
return backup_data
|
72
|
+
end
|
73
|
+
|
74
|
+
def save_backup_data()
|
75
|
+
open @backup_state_file_path, "w" do |h|
|
76
|
+
h.write(@backup_data.to_json)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def load_backup_data()
|
81
|
+
@backup_data = JSON.parse(open(@backup_state_file_path).read)
|
82
|
+
end
|
83
|
+
|
84
|
+
def ec2(key_id=@aws_access_key_id, key=@aws_secret_access_key, region=@region)
|
85
|
+
return RightAws::Ec2.new(key_id, key, {:region=>region})
|
86
|
+
end
|
87
|
+
|
88
|
+
def find_local_instance()
|
89
|
+
private_ip = UDPSocket.open {|s| s.connect("8.8.8.8", 1); s.addr.last}
|
90
|
+
ec2.describe_instances.each {|instance|
|
91
|
+
return instance if instance[:private_ip_address] == private_ip
|
92
|
+
}
|
93
|
+
return nil
|
94
|
+
end
|
95
|
+
|
96
|
+
def find_local_ebs(mount_point='/dev/sdh')
|
97
|
+
return nil if not local_instance = find_local_instance
|
98
|
+
local_instance[:block_device_mappings].each {|volume|
|
99
|
+
return volume if volume[:device_name] == mount_point
|
100
|
+
}
|
101
|
+
return nil
|
102
|
+
end
|
103
|
+
|
104
|
+
def get_snapshot(snapshot_id)
|
105
|
+
ec2.describe_snapshots {|a_snapshot|
|
106
|
+
return a_snapshot if a_snapshot[:aws_id] == snapshot_id
|
107
|
+
}
|
108
|
+
end
|
109
|
+
|
110
|
+
def create_local_snapshot(pre_snapshot_tasks=nil, post_snapshot_tasks=nil, mount_point='/dev/sdh')
|
111
|
+
local_instance=find_local_instance
|
112
|
+
ip = local_instance[:ip_address]
|
113
|
+
instance_id = local_instance[:aws_instance_id]
|
114
|
+
volume_id = local_instance[:block_device_mappings].select{|m| m[:device_name]==mount_point}.first[:ebs_volume_id]
|
115
|
+
return nil if not ebs = find_local_ebs(mount_point)
|
116
|
+
pre_snapshot_tasks.each do |task|
|
117
|
+
result = systemu(task)
|
118
|
+
unless result[0].exitstatus == 0
|
119
|
+
error_string = "Error while executing pre-snapshot task: #{task} on #{@drebs_host_name} #{ip}:#{mount_point} #{instance_id}:#{volume_id} #{result[1]} #{result[2]}"
|
120
|
+
@log.error(error_string)
|
121
|
+
send_email("DREBS Error!", error_string)
|
122
|
+
end
|
123
|
+
end if pre_snapshot_tasks
|
124
|
+
snapshot = ec2.create_snapshot(ebs[:ebs_volume_id], "DREBS #{@drebs_host_name} #{ip}:#{mount_point} #{instance_id}:#{volume_id}")
|
125
|
+
Thread.new(snapshot[:aws_id], post_snapshot_tasks) {|snapshot_id, post_snapshot_tasks|
|
126
|
+
1.upto(500) {|a|
|
127
|
+
sleep(3)
|
128
|
+
break if get_snapshot(snapshot_id)[:aws_status] == 'completed'
|
129
|
+
}
|
130
|
+
post_snapshot_tasks.each do |task|
|
131
|
+
result = systemu(task)
|
132
|
+
unless result[0].exitstatus == 0
|
133
|
+
error_string = "Error while executing post-snapshot task: #{task} on #{@drebs_host_name} #{ip}:#{mount_point} #{instance_id}:#{volume_id} #{result[1]} #{result[2]}"
|
134
|
+
@log.error(error_string)
|
135
|
+
send_email("DREBS Error!", error_string)
|
136
|
+
end
|
137
|
+
end if post_snapshot_tasks
|
138
|
+
}
|
139
|
+
return snapshot
|
140
|
+
end
|
141
|
+
|
142
|
+
def find_local_snapshots(mount_point='/dev/sdh')
|
143
|
+
return nil if not ebs = find_local_ebs(mount_point)
|
144
|
+
snapshots = []
|
145
|
+
ec2.describe_snapshots.each {|snapshot|
|
146
|
+
snapshots.push(snapshot) if snapshot[:aws_volume_id] == ebs[:ebs_volume_id]
|
147
|
+
}
|
148
|
+
return snapshots
|
149
|
+
end
|
150
|
+
|
151
|
+
def prune_backups(backup_data)
|
152
|
+
to_prune = {}
|
153
|
+
backup_data.collect {|a_backup_strategy|
|
154
|
+
if a_backup_strategy['previous_snapshots'].count > a_backup_strategy['num_to_keep']
|
155
|
+
to_prune[a_backup_strategy['previous_snapshots'].shift] = nil
|
156
|
+
end
|
157
|
+
}
|
158
|
+
to_prune.each_key {|snapshot_to_prune|
|
159
|
+
ec2.delete_snapshot(snapshot_to_prune) unless backup_data.any? {|a_backup_strategy|
|
160
|
+
a_backup_strategy['previous_snapshots'].include?(snapshot_to_prune)
|
161
|
+
}
|
162
|
+
}
|
163
|
+
end
|
164
|
+
|
165
|
+
def send_email(subject, body, options = {})
|
166
|
+
host = options[:email_host] || options['email_host'] || EMAIL_HOST
|
167
|
+
port = options[:email_port] || options['email_port'] || EMAIL_PORT
|
168
|
+
username = options[:email_username] || options['email_username'] || EMAIL_USERNAME
|
169
|
+
password = options[:email_password] || options['email_password'] || EMAIL_PASSWORD
|
170
|
+
|
171
|
+
|
172
|
+
msg = "Subject: #{subject}\n\n#{body}"
|
173
|
+
smtp = Net::SMTP.new 'smtp.gmail.com', 587
|
174
|
+
smtp.enable_starttls
|
175
|
+
smtp.start('gmail.com', username, password, :login) {|smtp|
|
176
|
+
smtp.send_message(msg, username, @email_on_exception)
|
177
|
+
}
|
178
|
+
end
|
179
|
+
|
180
|
+
def run_drebs_cron
|
181
|
+
begin
|
182
|
+
unless File.exists?(@backup_state_file_path)
|
183
|
+
@backup_data = setup_backup_data()
|
184
|
+
|
185
|
+
mount_points = @backup_data.map {|strategy| strategy['mount_point']}.compact.flatten.uniq
|
186
|
+
|
187
|
+
mount_points.map do |mount_point|
|
188
|
+
strategies = @backup_data.select{|strategy| strategy['mount_point'] == mount_point}
|
189
|
+
pre_snapshot_tasks = strategies.map {|strategy| strategy['pre_snapshot_tasks']}.compact.flatten.uniq
|
190
|
+
post_snapshot_tasks = strategies.map {|strategy| strategy['post_snapshot_tasks']}.compact.flatten.uniq
|
191
|
+
unless strategies.empty?
|
192
|
+
@log.info("creating snapshot of #{mount_point}")
|
193
|
+
snapshot = create_local_snapshot(pre_snapshot_tasks, post_snapshot_tasks, mount_point)
|
194
|
+
end
|
195
|
+
strategies.collect {|strategy|
|
196
|
+
strategy['previous_snapshots'].push(snapshot[:aws_id])
|
197
|
+
}
|
198
|
+
end
|
199
|
+
|
200
|
+
|
201
|
+
save_backup_data
|
202
|
+
else
|
203
|
+
load_backup_data
|
204
|
+
@backup_data.collect {|strategy|
|
205
|
+
strategy['hours_until_next_run'] -= 1
|
206
|
+
}
|
207
|
+
backup_now = @backup_data.collect {|strategy| strategy if strategy['hours_until_next_run'] <= 0}.compact
|
208
|
+
|
209
|
+
|
210
|
+
mount_points = @backup_data.map {|strategy| strategy['mount_point']}.compact.flatten.uniq
|
211
|
+
|
212
|
+
mount_points.map do |mount_point|
|
213
|
+
strategies = @backup_data.select{|strategy| strategy['mount_point'] == mount_point}
|
214
|
+
pre_snapshot_tasks = strategies.map {|strategy| strategy['pre_snapshot_tasks']}.compact.flatten.uniq
|
215
|
+
post_snapshot_tasks = strategies.map {|strategy| strategy['post_snapshot_tasks']}.compact.flatten.uniq
|
216
|
+
unless strategies.empty?
|
217
|
+
@log.info("creating snapshot of #{mount_point}")
|
218
|
+
snapshot = create_local_snapshot(pre_snapshot_tasks, post_snapshot_tasks, mount_point)
|
219
|
+
end
|
220
|
+
strategies.collect {|strategy|
|
221
|
+
strategy['previous_snapshots'].push(snapshot[:aws_id])
|
222
|
+
strategy['hours_until_next_run'] = strategy['hours_between']
|
223
|
+
}
|
224
|
+
|
225
|
+
end
|
226
|
+
|
227
|
+
prune_backups(@backup_data)
|
228
|
+
save_backup_data
|
229
|
+
end
|
230
|
+
rescue Exception => error
|
231
|
+
@log.error("Exception occured during backup: #{error.message}\n#{error.backtrace.join("\n")}")
|
232
|
+
send_email("DREBS Error! on #{@drebs_host_name}", "Host: #{@drebs_host_name} AWS Instance: #{find_local_instance[:aws_instance_id]}\n#{error.message}\n#{error.backtrace.join("\n")}")
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
end
|
237
|
+
|
238
|
+
if __FILE__ == $0
|
239
|
+
status = DATA.flock(File::LOCK_EX|File::LOCK_NB)
|
240
|
+
exit(42) unless status == 0
|
241
|
+
Main {
|
242
|
+
def run
|
243
|
+
drebs = DREBS.new
|
244
|
+
drebs.run_drebs_cron
|
245
|
+
end
|
246
|
+
}
|
247
|
+
end
|
248
|
+
|
249
|
+
__END__
|
data/drebs.gemspec
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
## drebs.gemspec
|
2
|
+
#
|
3
|
+
|
4
|
+
Gem::Specification::new do |spec|
|
5
|
+
spec.name = "drebs"
|
6
|
+
spec.version = "0.0.1"
|
7
|
+
spec.platform = Gem::Platform::RUBY
|
8
|
+
spec.summary = "drebs"
|
9
|
+
spec.description = "description: drebs kicks the ass"
|
10
|
+
|
11
|
+
spec.files =
|
12
|
+
["README.md",
|
13
|
+
"Rakefile",
|
14
|
+
"bin",
|
15
|
+
"bin/drebs",
|
16
|
+
"drebs.gemspec",
|
17
|
+
"lib",
|
18
|
+
"lib/drebs.rb"]
|
19
|
+
|
20
|
+
spec.executables = ["drebs"]
|
21
|
+
spec.require_path = "lib"
|
22
|
+
|
23
|
+
spec.test_files = nil
|
24
|
+
|
25
|
+
|
26
|
+
spec.add_dependency(*["right_aws", " >= 3.0.0 "])
|
27
|
+
|
28
|
+
spec.add_dependency(*["logger", " >= 1.2.8 "])
|
29
|
+
|
30
|
+
spec.add_dependency(*["main", " >= 5.0.0 "])
|
31
|
+
|
32
|
+
spec.add_dependency(*["systemu", " >= 2.4.2 "])
|
33
|
+
|
34
|
+
spec.add_dependency(*["json", " >= 1.5.1 "])
|
35
|
+
|
36
|
+
|
37
|
+
spec.extensions.push(*[])
|
38
|
+
|
39
|
+
spec.rubyforge_project = "DREBS"
|
40
|
+
spec.author = "Garett Shulman"
|
41
|
+
spec.email = "garett@dojo4.com"
|
42
|
+
spec.homepage = "https://github.com/dojo4/drebs"
|
43
|
+
end
|
data/lib/drebs.rb
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
# -*- encoding : utf-8 -*-
|
2
|
+
# built-ins
|
3
|
+
#
|
4
|
+
|
5
|
+
# DREBS libs
|
6
|
+
#
|
7
|
+
module DREBS
|
8
|
+
Version = '0.0.1' unless defined?(Version)
|
9
|
+
|
10
|
+
def version
|
11
|
+
DREBS::Version
|
12
|
+
end
|
13
|
+
|
14
|
+
def dependencies
|
15
|
+
{
|
16
|
+
'right_aws' => [ 'right_aws' , ' >= 3.0.0 ' ] ,
|
17
|
+
'logger' => [ 'logger' , ' >= 1.2.8 ' ] ,
|
18
|
+
'main' => [ 'main' , ' >= 5.0.0 ' ] ,
|
19
|
+
'systemu' => [ 'systemu' , ' >= 2.4.2 ' ] ,
|
20
|
+
'json' => [ 'json' , ' >= 1.5.1 ' ] ,
|
21
|
+
}
|
22
|
+
end
|
23
|
+
|
24
|
+
def libdir(*args, &block)
|
25
|
+
@libdir ||= File.expand_path(__FILE__).sub(/\.rb$/,'')
|
26
|
+
args.empty? ? @libdir : File.join(@libdir, *args)
|
27
|
+
ensure
|
28
|
+
if block
|
29
|
+
begin
|
30
|
+
$LOAD_PATH.unshift(@libdir)
|
31
|
+
block.call()
|
32
|
+
ensure
|
33
|
+
$LOAD_PATH.shift()
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def load(*libs)
|
39
|
+
libs = libs.join(' ').scan(/[^\s+]+/)
|
40
|
+
DREBS.libdir{ libs.each{|lib| Kernel.load(lib) } }
|
41
|
+
end
|
42
|
+
|
43
|
+
extend(DREBS)
|
44
|
+
end
|
45
|
+
Drebs = DREBS
|
46
|
+
|
47
|
+
# gems
|
48
|
+
#
|
49
|
+
begin
|
50
|
+
require 'rubygems'
|
51
|
+
rescue LoadError
|
52
|
+
nil
|
53
|
+
end
|
54
|
+
|
55
|
+
if defined?(gem)
|
56
|
+
DREBS.dependencies.each do |lib, dependency|
|
57
|
+
gem(*dependency)
|
58
|
+
require(lib)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
DREBS.load %w[
|
63
|
+
]
|
metadata
ADDED
@@ -0,0 +1,105 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: drebs
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Garett Shulman
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-03-01 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: right_aws
|
16
|
+
requirement: &2152789220 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 3.0.0
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *2152789220
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: logger
|
27
|
+
requirement: &2152788720 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 1.2.8
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *2152788720
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: main
|
38
|
+
requirement: &2152788240 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: 5.0.0
|
44
|
+
type: :runtime
|
45
|
+
prerelease: false
|
46
|
+
version_requirements: *2152788240
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: systemu
|
49
|
+
requirement: &2152787760 !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 2.4.2
|
55
|
+
type: :runtime
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: *2152787760
|
58
|
+
- !ruby/object:Gem::Dependency
|
59
|
+
name: json
|
60
|
+
requirement: &2152787280 !ruby/object:Gem::Requirement
|
61
|
+
none: false
|
62
|
+
requirements:
|
63
|
+
- - ! '>='
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: 1.5.1
|
66
|
+
type: :runtime
|
67
|
+
prerelease: false
|
68
|
+
version_requirements: *2152787280
|
69
|
+
description: ! 'description: drebs kicks the ass'
|
70
|
+
email: garett@dojo4.com
|
71
|
+
executables:
|
72
|
+
- drebs
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- README.md
|
77
|
+
- Rakefile
|
78
|
+
- bin/drebs
|
79
|
+
- drebs.gemspec
|
80
|
+
- lib/drebs.rb
|
81
|
+
homepage: https://github.com/dojo4/drebs
|
82
|
+
licenses: []
|
83
|
+
post_install_message:
|
84
|
+
rdoc_options: []
|
85
|
+
require_paths:
|
86
|
+
- lib
|
87
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
88
|
+
none: false
|
89
|
+
requirements:
|
90
|
+
- - ! '>='
|
91
|
+
- !ruby/object:Gem::Version
|
92
|
+
version: '0'
|
93
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
94
|
+
none: false
|
95
|
+
requirements:
|
96
|
+
- - ! '>='
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
version: '0'
|
99
|
+
requirements: []
|
100
|
+
rubyforge_project: DREBS
|
101
|
+
rubygems_version: 1.8.11
|
102
|
+
signing_key:
|
103
|
+
specification_version: 3
|
104
|
+
summary: drebs
|
105
|
+
test_files: []
|