mongo-ec2-backup 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +7 -0
- data/Gemfile.lock +37 -0
- data/LICENCE +24 -0
- data/README.markdown +68 -0
- data/Rakefile +37 -0
- data/bin/mongo_lock_and_snapshot +50 -0
- data/lib/ec2-consistent-backup.rb +169 -0
- data/lib/ec2_instance_identifier.rb +32 -0
- data/lib/ec2_snapshot_restorer.rb +34 -0
- data/lib/ec2_volume_snapshoter.rb +169 -0
- metadata +119 -0
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
GEM
|
2
|
+
remote: http://rubygems.org/
|
3
|
+
specs:
|
4
|
+
bson (1.3.1)
|
5
|
+
bson_ext (1.5.1)
|
6
|
+
builder (3.0.0)
|
7
|
+
excon (0.15.5)
|
8
|
+
fog (1.5.0)
|
9
|
+
builder
|
10
|
+
excon (~> 0.14)
|
11
|
+
formatador (~> 0.2.0)
|
12
|
+
mime-types
|
13
|
+
multi_json (~> 1.0)
|
14
|
+
net-scp (~> 1.0.4)
|
15
|
+
net-ssh (>= 2.1.3)
|
16
|
+
nokogiri (~> 1.5.0)
|
17
|
+
ruby-hmac
|
18
|
+
formatador (0.2.3)
|
19
|
+
mime-types (1.19)
|
20
|
+
mongo (1.3.1)
|
21
|
+
bson (>= 1.3.1)
|
22
|
+
multi_json (1.3.6)
|
23
|
+
net-scp (1.0.4)
|
24
|
+
net-ssh (>= 1.99.1)
|
25
|
+
net-ssh (2.5.2)
|
26
|
+
nokogiri (1.5.5)
|
27
|
+
ruby-hmac (0.4.0)
|
28
|
+
trollop (1.16.2)
|
29
|
+
|
30
|
+
PLATFORMS
|
31
|
+
ruby
|
32
|
+
|
33
|
+
DEPENDENCIES
|
34
|
+
bson_ext
|
35
|
+
fog
|
36
|
+
mongo
|
37
|
+
trollop
|
data/LICENCE
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# Copyright (c) 2011 Fotonauts
|
2
|
+
# All rights reserved.
|
3
|
+
# Redistribution and use in source and binary forms, with or without
|
4
|
+
# modification, are permitted provided that the following conditions are met:
|
5
|
+
#
|
6
|
+
# * Redistributions of source code must retain the above copyright
|
7
|
+
# notice, this list of conditions and the following disclaimer.
|
8
|
+
# * Redistributions in binary form must reproduce the above copyright
|
9
|
+
# notice, this list of conditions and the following disclaimer in the
|
10
|
+
# documentation and/or other materials provided with the distribution.
|
11
|
+
# * Neither the name of the Fotonauts, Fotopedia nor the
|
12
|
+
# names of its contributors may be used to endorse or promote products
|
13
|
+
# derived from this software without specific prior written permission.
|
14
|
+
#
|
15
|
+
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
|
16
|
+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
17
|
+
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
18
|
+
# DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
|
19
|
+
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
20
|
+
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
21
|
+
# OSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
22
|
+
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
23
|
+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
24
|
+
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
data/README.markdown
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
# Mongo consistent backup over RAID EBS disks on EC2 instance
|
2
|
+
|
3
|
+
Suite of tools to backup and manage snapshots of MongoDB data set to EC2 Snapshots.
|
4
|
+
|
5
|
+
## Lock and Snapshot: lock_and_snapshot.rb
|
6
|
+
|
7
|
+
### Usage
|
8
|
+
|
9
|
+
Snapshot a list of devices on a given instance on ec2. Requires network access in order to lock and unlock Mongo
|
10
|
+
|
11
|
+
```shell
|
12
|
+
./lock_and_snapshot.rb -a ACCESS_KEY_ID -s SECRET_ACCESS_KEY --hostname server01 --devices /dev/sdl,/dev/slm --type daily --limit 4
|
13
|
+
```
|
14
|
+
|
15
|
+
* --port, -p <i>: Mongo port to connect to (default: 27017)
|
16
|
+
* --access-key-id, -a <s>: Access Key Id for AWS
|
17
|
+
* --secret-access-key, -s <s>: Secret Access Key for AWS
|
18
|
+
* --devices, -d <s>: Devices to snapshot, comma separated
|
19
|
+
* --hostname, -h <s>: Hostname to look for. Should resolve to a local EC2 Ip
|
20
|
+
* --type, -t <s>: Snapshot type, to choose among snapshot,weekly,monthly,daily,yearly (default: snapshot)
|
21
|
+
* --limit, -l <i>: Cleanup old snapshots to keep only limit snapshots. Default values are stored in EC2VolumeSnapshoter::KIND
|
22
|
+
* --region: Region hosting the instances
|
23
|
+
* --help, -e: Show this message
|
24
|
+
|
25
|
+
### Usage in chef environment
|
26
|
+
|
27
|
+
In order to run the command from a remote server (the Chef server or any administrative node of your grid), you need to be able to know the lists of the devices you wish to snapshot.
|
28
|
+
|
29
|
+
By using the ohai-raid plugin (https://github.com/octplane/ohai-raid), Chef clients can fill part of their Chef registry with information about the software managed RAID arrays running.
|
30
|
+
This information can be fetched out for use at a later point via the knife script provided in the ohai-raid package:
|
31
|
+
|
32
|
+
```
|
33
|
+
knife exec scripts/show_raid_devices server01.fqdn.com /dev/md0
|
34
|
+
/dev/sdl,/dev/sdm,/dev/sdn,/dev/sdo
|
35
|
+
```
|
36
|
+
|
37
|
+
You can combine the two tools to automate daily backup of you MongoDB server:
|
38
|
+
|
39
|
+
```
|
40
|
+
./lock_and_snapshot.rb -a ACCESS_KEY_ID -s SECRET_ACCESS_KEY --hostname server01 --devices $(knife exec /path/to/scripts/show_raid_devices server01.fqdn.com /dev/md0) --type daily
|
41
|
+
```
|
42
|
+
|
43
|
+
### Tool Description
|
44
|
+
|
45
|
+
* Find instance id by resolving the hostname provided in the CLI and scanning the instances in EC2
|
46
|
+
* Lock Mongo by connecting via the hostname:port provided in the parameters
|
47
|
+
* Snapshot the disks, delete old backups
|
48
|
+
* Unlock Mongo
|
49
|
+
|
50
|
+
## MD inspection: ec2-consistent-backup.rb
|
51
|
+
|
52
|
+
### Usage
|
53
|
+
|
54
|
+
This script demonstrates the way it analyses Mongo DB Data path to extract the MD device and components associated
|
55
|
+
|
56
|
+
```shell
|
57
|
+
./ec2-consistent-backup -p 27017
|
58
|
+
```
|
59
|
+
|
60
|
+
### Tool description
|
61
|
+
|
62
|
+
* connect to mongo at port provided, retrieves dbpath
|
63
|
+
* find what mount this dbpath corresponds to
|
64
|
+
* use /proc/mdstat to find out which drive are corresponding to the dbpath mount disk
|
65
|
+
|
66
|
+
# API
|
67
|
+
|
68
|
+
Internal API documentation is at: http://rubydoc.info/github/octplane/mongo-ec2-consistent-backup/master/frames
|
data/Rakefile
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'rake'
|
2
|
+
require "rake/clean"
|
3
|
+
require 'rake/gempackagetask'
|
4
|
+
require 'rake/rdoctask'
|
5
|
+
|
6
|
+
desc "Packages up Swissr."
|
7
|
+
task :default => :package
|
8
|
+
|
9
|
+
spec = Gem::Specification.new do |s|
|
10
|
+
s.name = 'mongo-ec2-backup'
|
11
|
+
s.version = '0.0.4'
|
12
|
+
s.summary = 'Snapshot your mongodb in the EC2 cloud via XFS Freeze'
|
13
|
+
|
14
|
+
s.author = 'Pierre Baillet'
|
15
|
+
s.email = 'oct@fotopedia.com'
|
16
|
+
s.homepage = 'https://github.com/octplane/mongo-ec2-consistent-backup'
|
17
|
+
|
18
|
+
# These dependencies are only for people who work on this gem
|
19
|
+
s.add_dependency 'fog'
|
20
|
+
s.add_dependency 'bson_ext'
|
21
|
+
s.add_dependency 'trollop'
|
22
|
+
s.add_dependency 'mongo'
|
23
|
+
|
24
|
+
# Include everything in the lib folder
|
25
|
+
s.files = FileList['lib/**/*.rb', 'bin/*', '[A-Z]*', 'test/**/*'].to_a
|
26
|
+
|
27
|
+
s.executables << "mongo_lock_and_snapshot"
|
28
|
+
|
29
|
+
# Supress the warning about no rubyforge project
|
30
|
+
s.rubyforge_project = 'nowarning'
|
31
|
+
end
|
32
|
+
|
33
|
+
Rake::GemPackageTask.new(spec) do |package|
|
34
|
+
package.gem_spec = spec
|
35
|
+
# package.need_tar = true
|
36
|
+
# package.need_zip = true
|
37
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# Lock a set of disk via the mongo lock command and snapshot them to the cloud
|
3
|
+
|
4
|
+
require 'rubygems'
|
5
|
+
ENV['BUNDLE_GEMFILE'] = File.join(File.dirname(__FILE__), "..", "Gemfile")
|
6
|
+
require 'bundler/setup'
|
7
|
+
require 'trollop'
|
8
|
+
|
9
|
+
$: << File.join("..", File.dirname(__FILE__), "lib")
|
10
|
+
require 'ec2-consistent-backup'
|
11
|
+
require 'ec2_volume_snapshoter'
|
12
|
+
|
13
|
+
opts = Trollop::options do
|
14
|
+
opt :path, "Data path to freeze", :type => :string, :required => true
|
15
|
+
opt :access_key_id, "Access Key Id for AWS", :type => :string, :required => true
|
16
|
+
opt :secret_access_key, "Secret Access Key for AWS", :type => :string, :required => true
|
17
|
+
opt :devices, "Devices to snapshot, comma separated", :type => :string, :required => true
|
18
|
+
opt :type, "Snapshot type, to choose among #{EC2VolumeSnapshoter::KINDS.keys.join(",")}", :default => "snapshot"
|
19
|
+
opt :limit, "Cleanup old snapshots to keep only limit snapshots", :type => :integer
|
20
|
+
end
|
21
|
+
|
22
|
+
# find instance id by
|
23
|
+
# - resolving name to ip
|
24
|
+
# - looking in EC2 for server
|
25
|
+
# Lock Mongo
|
26
|
+
# Snapshot
|
27
|
+
# Unlock
|
28
|
+
|
29
|
+
aki = opts[:access_key_id]
|
30
|
+
sak = opts[:secret_access_key]
|
31
|
+
path = opts[:path]
|
32
|
+
|
33
|
+
`/usr/sbin/xfs_freeze -f #{path}`
|
34
|
+
|
35
|
+
begin
|
36
|
+
snapshoter = EC2VolumeSnapshoter.new(aki, sak)
|
37
|
+
limit = if opts[:limit] == nil
|
38
|
+
EC2VolumeSnapshoter::KINDS[opts[:type]]
|
39
|
+
else
|
40
|
+
opts[:limit]
|
41
|
+
end
|
42
|
+
|
43
|
+
snapshoter.snapshot_devices(opts[:devices].split(/,/), "Mongo Snapshot", opts[:type], limit)
|
44
|
+
rescue Exception => e
|
45
|
+
require "pp"
|
46
|
+
puts e.inspect
|
47
|
+
pp e.backtrace
|
48
|
+
ensure
|
49
|
+
`/usr/sbin/xfs_freeze -u #{path}`
|
50
|
+
end
|
@@ -0,0 +1,169 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'mongo'
|
3
|
+
require 'open-uri'
|
4
|
+
|
5
|
+
=begin
|
6
|
+
- check S3 credentials
|
7
|
+
- check disk location of data
|
8
|
+
- check this is on a remotely mounted disk (or md drive)
|
9
|
+
- http://www.mongodb.org/display/DOCS/getCmdLineOpts+command
|
10
|
+
=end
|
11
|
+
|
12
|
+
|
13
|
+
=begin
|
14
|
+
/proc/mdstat content:
|
15
|
+
|
16
|
+
Personalities : [raid0]
|
17
|
+
md0 : active raid0 sdo[3] sdn[2] sdm[1] sdl[0]
|
18
|
+
838859776 blocks 256k chunks
|
19
|
+
|
20
|
+
unused devices: <none>
|
21
|
+
=end
|
22
|
+
class NoSuchSetException < Exception; end
|
23
|
+
# Parse the existing RAID sets by reading /prod/mdstat
|
24
|
+
# Cheap alternative to using FFI to interface with libdm
|
25
|
+
class MDInspector
|
26
|
+
MDFILE = "/proc/mdstat"
|
27
|
+
PERSONALITIES = "Personalities :"
|
28
|
+
attr_reader :has_md, :personalities
|
29
|
+
attr_reader :drives
|
30
|
+
def initialize(mdfile = MDFILE)
|
31
|
+
@has_md = false
|
32
|
+
if File.exists?(mdfile)
|
33
|
+
stat_data = File.open(mdfile).read.split(/\n/)
|
34
|
+
personalities_line = stat_data.grep(/#{PERSONALITIES}/)
|
35
|
+
if personalities_line =~ /#{PERSONALITIES}(.+)/
|
36
|
+
@personalities = $1
|
37
|
+
else
|
38
|
+
@has_md = false
|
39
|
+
end
|
40
|
+
@set_metadata = {}
|
41
|
+
stat_data.grep(/^md[0-9]+ : /).each do |md_info|
|
42
|
+
if md_info =~ /^md([0-9]+) : active ([^ ]+) (.*)$/
|
43
|
+
set_name = "md#{$1}"
|
44
|
+
personality = $2
|
45
|
+
drives = $3.split(/ /).map{ |i| "/dev/"+i.gsub(/\[[0-9]+\]/,'') }.to_a
|
46
|
+
@set_metadata[set_name] = { :set_name=> set_name, :personality => personality,
|
47
|
+
:drives => drives}
|
48
|
+
end
|
49
|
+
end
|
50
|
+
@has_md = true if @set_metadata.keys.length > 0
|
51
|
+
end
|
52
|
+
end
|
53
|
+
# Returns the information about the MD set @name
|
54
|
+
def set(name)
|
55
|
+
# Handle "/dev/foobar" instead of "foobar"
|
56
|
+
if name =~ /\/dev\/(.*)$/
|
57
|
+
name = $1
|
58
|
+
end
|
59
|
+
return @set_metadata[name] if @set_metadata.has_key?(name)
|
60
|
+
raise NoSuchSetException.new(name)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
class NotMountedException < Exception; end
|
65
|
+
class MountInspector
|
66
|
+
def initialize(file = '/etc/mtab')
|
67
|
+
@dev_to_fs = {}
|
68
|
+
@fs_to_dev = {}
|
69
|
+
File.open(file).read.split(/\n/).map {|line| line.split(/ /)[0..1]}.each do |m|
|
70
|
+
@dev_to_fs[m[0]] = m[1] if m[0] != "none"
|
71
|
+
@fs_to_dev[m[1]] = m[0] if m[1] != "none"
|
72
|
+
end
|
73
|
+
end
|
74
|
+
def where_is_mounted(device)
|
75
|
+
return @dev_to_fs[device] if @dev_to_fs.has_key?(device)
|
76
|
+
raise NotMountedException.new(device)
|
77
|
+
end
|
78
|
+
def which_device(folder)
|
79
|
+
# Level 0 optimisation+ Handle "/" folder
|
80
|
+
return @fs_to_dev[folder] if @fs_to_dev.has_key?(folder)
|
81
|
+
|
82
|
+
components = folder.split(/\//)
|
83
|
+
components.size.downto(0).each do |sz|
|
84
|
+
current_folder = components[0..sz-1].join("/")
|
85
|
+
current_folder = "/" if current_folder == ""
|
86
|
+
return @fs_to_dev[current_folder] if @fs_to_dev.has_key?(current_folder)
|
87
|
+
end
|
88
|
+
raise NotMountedException.new(folder)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
module MongoHelper
|
93
|
+
class DataLocker
|
94
|
+
attr_reader :path
|
95
|
+
def initialize(port = 27017, host = 'localhost')
|
96
|
+
@m = Mongo::Connection.new(host, port)
|
97
|
+
args = @m['admin'].command({'getCmdLineOpts' => 1 })['argv']
|
98
|
+
p = args.index('--dbpath')
|
99
|
+
@path = args[p+1]
|
100
|
+
@path = File.readlink(@path) if File.symlink?(@path)
|
101
|
+
|
102
|
+
end
|
103
|
+
def lock
|
104
|
+
return if locked?
|
105
|
+
@m.lock!
|
106
|
+
while !locked? do
|
107
|
+
sleep(1)
|
108
|
+
end
|
109
|
+
raise "Not locked as asked" if !locked?
|
110
|
+
end
|
111
|
+
def locked?
|
112
|
+
@m.locked?
|
113
|
+
end
|
114
|
+
def unlock
|
115
|
+
return if !locked?
|
116
|
+
raise "Already unlocked" if !locked?
|
117
|
+
@m.unlock!
|
118
|
+
while locked? do
|
119
|
+
sleep(1)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
def log s
|
126
|
+
$stderr.puts "[#{Time.now}]: #{s}"
|
127
|
+
end
|
128
|
+
|
129
|
+
if __FILE__ == $0
|
130
|
+
require 'trollop'
|
131
|
+
opts = Trollop::options do
|
132
|
+
opt :port, "Mongo port to connect to", :default => 27017
|
133
|
+
end
|
134
|
+
|
135
|
+
# First connect to mongo and find the dbpath
|
136
|
+
port = opts[:port]
|
137
|
+
m = MongoHelper::DataLocker.new(port)
|
138
|
+
data_location = m.path
|
139
|
+
log "Mongo at #{port} has its data in #{data_location}."
|
140
|
+
|
141
|
+
|
142
|
+
mount_inspector = MountInspector.new
|
143
|
+
raid_set = mount_inspector.which_device(data_location)
|
144
|
+
log "This path is on the device #{raid_set}."
|
145
|
+
|
146
|
+
begin
|
147
|
+
raid_sets = MDInspector.new
|
148
|
+
drives = raid_sets.set(raid_set)[:drives]
|
149
|
+
|
150
|
+
log "This device is the MD device built with #{drives.inspect}."
|
151
|
+
|
152
|
+
# # this code probably works, but is way to dangerous.
|
153
|
+
# m.lock
|
154
|
+
# begin
|
155
|
+
# log "Locked mongo"
|
156
|
+
# e = EC2VolumeSnapshoter.new(opts[:access_key_id], opts[:secret_access_key], opts[:region])
|
157
|
+
# e.snapshot_devices(drives)
|
158
|
+
# rescue Exception => e
|
159
|
+
# puts e.inspect
|
160
|
+
# ensure
|
161
|
+
# m.unlock
|
162
|
+
# log "Unlocked mongo"
|
163
|
+
# end
|
164
|
+
|
165
|
+
rescue NoSuchSetException => e
|
166
|
+
log "Device #{raid_set} is not a MD device, bailing out"
|
167
|
+
raise e
|
168
|
+
end
|
169
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'resolv'
|
2
|
+
require 'fog'
|
3
|
+
|
4
|
+
class EC2InstanceNotFoundException < Exception; end
|
5
|
+
# Fetch an instance from its private ip address
|
6
|
+
class EC2InstanceIdentifier
|
7
|
+
# Need access_key_id, secret_access_key
|
8
|
+
def initialize(aki, sak)
|
9
|
+
@compute = Fog::Compute.new({:provider => 'AWS', :aws_access_key_id => aki, :aws_secret_access_key => sak})
|
10
|
+
end
|
11
|
+
# Returns the instance corresponding to the provided hostname
|
12
|
+
def get_instance(hostname)
|
13
|
+
ip = Resolv.getaddress(hostname)
|
14
|
+
instance = @compute.servers.all().find { |i| i.private_ip_address == ip || i.public_ip_address == ip }
|
15
|
+
raise InstanceNotFoundException.new(hostname) if ! instance
|
16
|
+
return instance
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
if __FILE__ == $0
|
21
|
+
require 'trollop'
|
22
|
+
require 'pp'
|
23
|
+
opts = Trollop::options do
|
24
|
+
opt :access_key_id, "Access Key Id for AWS", :type => :string, :required => true
|
25
|
+
opt :secret_access_key, "Secret Access Key for AWS", :type => :string, :required => true
|
26
|
+
opt :hostname, "Hostname to look for. Should resolve to a local EC2 Ip", :type => :string, :required => true
|
27
|
+
end
|
28
|
+
|
29
|
+
eii = EC2InstanceIdentifier.new(opts[:access_key_id], opts[:secret_access_key])
|
30
|
+
pp eii.get_instance(opts[:hostname])
|
31
|
+
|
32
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'fog'
|
2
|
+
require 'open-uri'
|
3
|
+
|
4
|
+
|
5
|
+
#
|
6
|
+
class SnapshotRestorer
|
7
|
+
def initialize(aki, sak, snap_ids)
|
8
|
+
@compute = Fog::Compute.new({:provider => 'AWS', :aws_access_key_id => aki, :aws_secret_access_key => sak})
|
9
|
+
@snaps = snap_ids
|
10
|
+
@volumes = []
|
11
|
+
end
|
12
|
+
def restore()
|
13
|
+
@snaps.each do | resource_id |
|
14
|
+
snap = @compute.snapshots.get(resource_id)
|
15
|
+
# Snap have the following tags
|
16
|
+
# application
|
17
|
+
# device
|
18
|
+
# instance_id
|
19
|
+
# date
|
20
|
+
# kind
|
21
|
+
|
22
|
+
t = snap.tags
|
23
|
+
volume = @compute.volumes.new :snapshot_id => snap.id, :size => snap.volume_size, :availability_zone => 'us-east-1c'
|
24
|
+
@compute.tags.create(:resource_id => volume.id, :key =>"application", :value => NAME_PREFIX)
|
25
|
+
@compute.tags.create(:resource_id => volume.id, :key =>"device", :value => device)
|
26
|
+
@compute.tags.create(:resource_id => volume.id, :key =>"date", :value => ts)
|
27
|
+
@compute.tags.create(:resource_id => volume.id, :key =>"kind", :value => kind)
|
28
|
+
volume.save
|
29
|
+
@volumes << volume
|
30
|
+
end
|
31
|
+
end
|
32
|
+
def connect(instance_id = open("http://169.254.169.254/latest/meta-data/instance-id").read)
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,169 @@
|
|
1
|
+
require 'fog'
|
2
|
+
require 'open-uri'
|
3
|
+
|
4
|
+
# This class is responsible of the snapshoting of given disks to EC2
|
5
|
+
# EC2 related permissions in IAM
|
6
|
+
# Sid": "Stmt1344254048404",
|
7
|
+
# "Action": [
|
8
|
+
# "ec2:CreateSnapshot",
|
9
|
+
# "ec2:DeleteSnapshot",
|
10
|
+
# "ec2:DescribeSnapshots",
|
11
|
+
# "ec2:CreateTags",
|
12
|
+
# "ec2:DescribeTags",
|
13
|
+
# "ec2:DescribeVolumes"
|
14
|
+
# ],
|
15
|
+
# "Effect": "Allow",
|
16
|
+
# "Resource": [
|
17
|
+
# "*"
|
18
|
+
# ]
|
19
|
+
#
|
20
|
+
|
21
|
+
class NoSuchVolumeException < Exception
|
22
|
+
def initialize(instance, volume, details)
|
23
|
+
@instance, @volume, @details = instance, volume, details
|
24
|
+
end
|
25
|
+
def to_s
|
26
|
+
"Unable to locate volume \"#{@volume}\" on #{@instance}\nKnow volumes for this instance are:\n#{@details.inspect}"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def log s
|
31
|
+
$stderr.puts "[#{Time.now}]: #{s}"
|
32
|
+
end
|
33
|
+
|
34
|
+
class EC2VolumeSnapshoter
|
35
|
+
NAME_PREFIX='Volume Snapshot'
|
36
|
+
|
37
|
+
# Kind of snapshot and their expiration in days
|
38
|
+
KINDS = { 'test' => 1,
|
39
|
+
'snapshot' => 0,
|
40
|
+
'daily' => 7,
|
41
|
+
'weekly' => 31,
|
42
|
+
'monthly' => 300,
|
43
|
+
'yearly' => 0}
|
44
|
+
|
45
|
+
attr_reader :instance_id
|
46
|
+
# Need access_key_id, secret_access_key and instance_id
|
47
|
+
# If not provided, attempt to fetch current instance_id
|
48
|
+
def initialize(aki, sak, instance_id = open("http://169.254.169.254/latest/meta-data/instance-id").read)
|
49
|
+
|
50
|
+
@instance_id = instance_id
|
51
|
+
|
52
|
+
@compute = Fog::Compute.new({:provider => 'AWS', :aws_access_key_id => aki, :aws_secret_access_key => sak})
|
53
|
+
end
|
54
|
+
# Snapshots the list of devices
|
55
|
+
# devices is an array of device attached to the instance (/dev/foo)
|
56
|
+
# name if the name of the snapshot
|
57
|
+
def snapshot_devices(devices, name = "#{instance_id}", kind = "test", limit = KINDS[kind])
|
58
|
+
log "Snapshot of kind #{kind}, limit set to #{limit} (0 means never purge)"
|
59
|
+
ts = DateTime.now.to_s
|
60
|
+
name = "#{NAME_PREFIX}:" + name
|
61
|
+
volumes = {}
|
62
|
+
devices.each do |device|
|
63
|
+
volumes[device] = find_volume_for_device(device)
|
64
|
+
end
|
65
|
+
sn = []
|
66
|
+
volumes.each do |device, volume|
|
67
|
+
log "Creating volume snapshot for #{device} on instance #{instance_id}"
|
68
|
+
snapshot = volume.snapshots.new
|
69
|
+
snapshot.description = name+" #{device}"
|
70
|
+
snapshot.save
|
71
|
+
sn << snapshot
|
72
|
+
snapshot.reload
|
73
|
+
|
74
|
+
@compute.tags.create(:resource_id => snapshot.id, :key =>"application", :value => NAME_PREFIX)
|
75
|
+
@compute.tags.create(:resource_id => snapshot.id, :key =>"device", :value => device)
|
76
|
+
@compute.tags.create(:resource_id => snapshot.id, :key =>"instance_id", :value =>instance_id)
|
77
|
+
@compute.tags.create(:resource_id => snapshot.id, :key =>"date", :value => ts)
|
78
|
+
@compute.tags.create(:resource_id => snapshot.id, :key =>"kind", :value => kind)
|
79
|
+
|
80
|
+
end
|
81
|
+
log "Waiting for snapshots to complete."
|
82
|
+
sn.each do |s|
|
83
|
+
begin
|
84
|
+
sleep(3)
|
85
|
+
s.reload
|
86
|
+
end while s.state == 'nil' || s.state == 'pending'
|
87
|
+
end
|
88
|
+
|
89
|
+
if limit != 0
|
90
|
+
# populate data structure with updated information
|
91
|
+
snapshots = list_snapshots(devices, kind)
|
92
|
+
nsnaps = snapshots.keys.length
|
93
|
+
if nsnaps-limit > 0
|
94
|
+
dates = snapshots.keys.sort
|
95
|
+
puts dates.inspect
|
96
|
+
extra_snapshots = dates[0..-limit]
|
97
|
+
remaining_snapshots = dates[-limit..-1]
|
98
|
+
extra_snapshots.each do |date|
|
99
|
+
snapshots[date].each do |snap|
|
100
|
+
log "Destroying #{snap.description} #{snap.id}"
|
101
|
+
snap.destroy
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
# List snapshots for a set of device and a given kind
|
109
|
+
require 'pp'
|
110
|
+
def list_snapshots(devices, kind)
|
111
|
+
volume_map = []
|
112
|
+
snapshots = {}
|
113
|
+
|
114
|
+
tags = @compute.tags.all(:key => 'instance_id', :value => instance_id)
|
115
|
+
tags.each do |tag|
|
116
|
+
snap = @compute.snapshots.get(tag.resource_id)
|
117
|
+
t = snap.tags
|
118
|
+
|
119
|
+
if devices.include?(t['device']) &&
|
120
|
+
instance_id == t['instance_id'] &&
|
121
|
+
NAME_PREFIX == t['application'] &&
|
122
|
+
kind == t['kind']
|
123
|
+
snapshots[t['date']] ||= []
|
124
|
+
snapshots[t['date']] << snap
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
# take out incomplete backups
|
129
|
+
snapshots.delete_if{ |date, snaps| snaps.length != devices.length }
|
130
|
+
snapshots
|
131
|
+
end
|
132
|
+
|
133
|
+
|
134
|
+
def find_volume_for_device(device)
|
135
|
+
my = []
|
136
|
+
@compute.volumes.all().each do |volume|
|
137
|
+
if volume.server_id == @instance_id
|
138
|
+
my << volume
|
139
|
+
if volume.device == device
|
140
|
+
return volume
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
raise NoSuchVolumeException.new(@instance_id, device, my)
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
if __FILE__ == $0
|
149
|
+
require 'trollop'
|
150
|
+
require 'pp'
|
151
|
+
|
152
|
+
opts = Trollop::options do
|
153
|
+
opt :access_key_id, "Access Key Id for AWS", :type => :string, :required => true
|
154
|
+
opt :secret_access_key, "Secret Access Key for AWS", :type => :string, :required => true
|
155
|
+
opt :instance_id, "Instance identifier", :type => :string, :required => true
|
156
|
+
opt :find_volume_for, "Show information for device path (mount point)", :type => :string
|
157
|
+
opt :snapshot, "Snapshot device path (mount point)", :type => :string
|
158
|
+
opt :snapshot_type, "Kind of snapshot (any of #{EC2VolumeSnapshoter::KINDS.keys.join(", ")})", :default => 'test'
|
159
|
+
|
160
|
+
end
|
161
|
+
|
162
|
+
evs = EC2VolumeSnapshoter.new(opts[:access_key_id], opts[:secret_access_key], opts[:instance_id])
|
163
|
+
if opts[:find_volume_for]
|
164
|
+
pp evs.find_volume_for_device(opts[:find_volume_for])
|
165
|
+
end
|
166
|
+
if opts[:snapshot]
|
167
|
+
evs.snapshot_devices([opts[:snapshot]])
|
168
|
+
end
|
169
|
+
end
|
metadata
ADDED
@@ -0,0 +1,119 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: mongo-ec2-backup
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.4
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Pierre Baillet
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-08-06 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fog
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: bson_ext
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: trollop
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: mongo
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :runtime
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
description:
|
79
|
+
email: oct@fotopedia.com
|
80
|
+
executables:
|
81
|
+
- mongo_lock_and_snapshot
|
82
|
+
extensions: []
|
83
|
+
extra_rdoc_files: []
|
84
|
+
files:
|
85
|
+
- lib/ec2-consistent-backup.rb
|
86
|
+
- lib/ec2_instance_identifier.rb
|
87
|
+
- lib/ec2_snapshot_restorer.rb
|
88
|
+
- lib/ec2_volume_snapshoter.rb
|
89
|
+
- bin/mongo_lock_and_snapshot
|
90
|
+
- Gemfile
|
91
|
+
- Gemfile.lock
|
92
|
+
- LICENCE
|
93
|
+
- Rakefile
|
94
|
+
- README.markdown
|
95
|
+
homepage: https://github.com/octplane/mongo-ec2-consistent-backup
|
96
|
+
licenses: []
|
97
|
+
post_install_message:
|
98
|
+
rdoc_options: []
|
99
|
+
require_paths:
|
100
|
+
- lib
|
101
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
102
|
+
none: false
|
103
|
+
requirements:
|
104
|
+
- - ! '>='
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: '0'
|
107
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
108
|
+
none: false
|
109
|
+
requirements:
|
110
|
+
- - ! '>='
|
111
|
+
- !ruby/object:Gem::Version
|
112
|
+
version: '0'
|
113
|
+
requirements: []
|
114
|
+
rubyforge_project: nowarning
|
115
|
+
rubygems_version: 1.8.23
|
116
|
+
signing_key:
|
117
|
+
specification_version: 3
|
118
|
+
summary: Snapshot your mongodb in the EC2 cloud via XFS Freeze
|
119
|
+
test_files: []
|