mongo-ec2-backup 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +7 -0
- data/Gemfile.lock +37 -0
- data/LICENCE +24 -0
- data/README.markdown +68 -0
- data/Rakefile +37 -0
- data/bin/mongo_lock_and_snapshot +50 -0
- data/lib/ec2-consistent-backup.rb +169 -0
- data/lib/ec2_instance_identifier.rb +32 -0
- data/lib/ec2_snapshot_restorer.rb +34 -0
- data/lib/ec2_volume_snapshoter.rb +169 -0
- metadata +119 -0
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
GEM
|
2
|
+
remote: http://rubygems.org/
|
3
|
+
specs:
|
4
|
+
bson (1.3.1)
|
5
|
+
bson_ext (1.5.1)
|
6
|
+
builder (3.0.0)
|
7
|
+
excon (0.15.5)
|
8
|
+
fog (1.5.0)
|
9
|
+
builder
|
10
|
+
excon (~> 0.14)
|
11
|
+
formatador (~> 0.2.0)
|
12
|
+
mime-types
|
13
|
+
multi_json (~> 1.0)
|
14
|
+
net-scp (~> 1.0.4)
|
15
|
+
net-ssh (>= 2.1.3)
|
16
|
+
nokogiri (~> 1.5.0)
|
17
|
+
ruby-hmac
|
18
|
+
formatador (0.2.3)
|
19
|
+
mime-types (1.19)
|
20
|
+
mongo (1.3.1)
|
21
|
+
bson (>= 1.3.1)
|
22
|
+
multi_json (1.3.6)
|
23
|
+
net-scp (1.0.4)
|
24
|
+
net-ssh (>= 1.99.1)
|
25
|
+
net-ssh (2.5.2)
|
26
|
+
nokogiri (1.5.5)
|
27
|
+
ruby-hmac (0.4.0)
|
28
|
+
trollop (1.16.2)
|
29
|
+
|
30
|
+
PLATFORMS
|
31
|
+
ruby
|
32
|
+
|
33
|
+
DEPENDENCIES
|
34
|
+
bson_ext
|
35
|
+
fog
|
36
|
+
mongo
|
37
|
+
trollop
|
data/LICENCE
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# Copyright (c) 2011 Fotonauts
|
2
|
+
# All rights reserved.
|
3
|
+
# Redistribution and use in source and binary forms, with or without
|
4
|
+
# modification, are permitted provided that the following conditions are met:
|
5
|
+
#
|
6
|
+
# * Redistributions of source code must retain the above copyright
|
7
|
+
# notice, this list of conditions and the following disclaimer.
|
8
|
+
# * Redistributions in binary form must reproduce the above copyright
|
9
|
+
# notice, this list of conditions and the following disclaimer in the
|
10
|
+
# documentation and/or other materials provided with the distribution.
|
11
|
+
# * Neither the name of the Fotonauts, Fotopedia nor the
|
12
|
+
# names of its contributors may be used to endorse or promote products
|
13
|
+
# derived from this software without specific prior written permission.
|
14
|
+
#
|
15
|
+
# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
|
16
|
+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
17
|
+
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
18
|
+
# DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
|
19
|
+
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
20
|
+
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
21
|
+
# OSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
22
|
+
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
23
|
+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
24
|
+
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
data/README.markdown
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
# Mongo consistent backup over RAID EBS disks on EC2 instance
|
2
|
+
|
3
|
+
Suite of tools to backup and manage snapshots of MongoDB data set to EC2 Snapshots.
|
4
|
+
|
5
|
+
## Lock and Snapshot: lock_and_snapshot.rb
|
6
|
+
|
7
|
+
### Usage
|
8
|
+
|
9
|
+
Snapshot a list of devices on a given instance on ec2. Requires network access in order to lock and unlock Mongo
|
10
|
+
|
11
|
+
```shell
|
12
|
+
./lock_and_snapshot.rb -a ACCESS_KEY_ID -s SECRET_ACCESS_KEY --hostname server01 --devices /dev/sdl,/dev/slm --type daily --limit 4
|
13
|
+
```
|
14
|
+
|
15
|
+
* --port, -p <i>: Mongo port to connect to (default: 27017)
|
16
|
+
* --access-key-id, -a <s>: Access Key Id for AWS
|
17
|
+
* --secret-access-key, -s <s>: Secret Access Key for AWS
|
18
|
+
* --devices, -d <s>: Devices to snapshot, comma separated
|
19
|
+
* --hostname, -h <s>: Hostname to look for. Should resolve to a local EC2 Ip
|
20
|
+
* --type, -t <s>: Snapshot type, to choose among snapshot,weekly,monthly,daily,yearly (default: snapshot)
|
21
|
+
* --limit, -l <i>: Cleanup old snapshots to keep only limit snapshots. Default values are stored in EC2VolumeSnapshoter::KIND
|
22
|
+
* --region: Region hosting the instances
|
23
|
+
* --help, -e: Show this message
|
24
|
+
|
25
|
+
### Usage in chef environment
|
26
|
+
|
27
|
+
In order to run the command from a remote server (the Chef server or any administrative node of your grid), you need to be able to know the lists of the devices you wish to snapshot.
|
28
|
+
|
29
|
+
By using the ohai-raid plugin (https://github.com/octplane/ohai-raid), Chef clients can fill part of their Chef registry with information about the software managed RAID arrays running.
|
30
|
+
This information can be fetched out for use at a later point via the knife script provided in the ohai-raid package:
|
31
|
+
|
32
|
+
```
|
33
|
+
knife exec scripts/show_raid_devices server01.fqdn.com /dev/md0
|
34
|
+
/dev/sdl,/dev/sdm,/dev/sdn,/dev/sdo
|
35
|
+
```
|
36
|
+
|
37
|
+
You can combine the two tools to automate daily backup of you MongoDB server:
|
38
|
+
|
39
|
+
```
|
40
|
+
./lock_and_snapshot.rb -a ACCESS_KEY_ID -s SECRET_ACCESS_KEY --hostname server01 --devices $(knife exec /path/to/scripts/show_raid_devices server01.fqdn.com /dev/md0) --type daily
|
41
|
+
```
|
42
|
+
|
43
|
+
### Tool Description
|
44
|
+
|
45
|
+
* Find instance id by resolving the hostname provided in the CLI and scanning the instances in EC2
|
46
|
+
* Lock Mongo by connecting via the hostname:port provided in the parameters
|
47
|
+
* Snapshot the disks, delete old backups
|
48
|
+
* Unlock Mongo
|
49
|
+
|
50
|
+
## MD inspection: ec2-consistent-backup.rb
|
51
|
+
|
52
|
+
### Usage
|
53
|
+
|
54
|
+
This script demonstrates the way it analyses Mongo DB Data path to extract the MD device and components associated
|
55
|
+
|
56
|
+
```shell
|
57
|
+
./ec2-consistent-backup -p 27017
|
58
|
+
```
|
59
|
+
|
60
|
+
### Tool description
|
61
|
+
|
62
|
+
* connect to mongo at port provided, retrieves dbpath
|
63
|
+
* find what mount this dbpath corresponds to
|
64
|
+
* use /proc/mdstat to find out which drive are corresponding to the dbpath mount disk
|
65
|
+
|
66
|
+
# API
|
67
|
+
|
68
|
+
Internal API documentation is at: http://rubydoc.info/github/octplane/mongo-ec2-consistent-backup/master/frames
|
data/Rakefile
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'rake'
|
2
|
+
require "rake/clean"
|
3
|
+
require 'rake/gempackagetask'
|
4
|
+
require 'rake/rdoctask'
|
5
|
+
|
6
|
+
desc "Packages up Swissr."
|
7
|
+
task :default => :package
|
8
|
+
|
9
|
+
spec = Gem::Specification.new do |s|
|
10
|
+
s.name = 'mongo-ec2-backup'
|
11
|
+
s.version = '0.0.4'
|
12
|
+
s.summary = 'Snapshot your mongodb in the EC2 cloud via XFS Freeze'
|
13
|
+
|
14
|
+
s.author = 'Pierre Baillet'
|
15
|
+
s.email = 'oct@fotopedia.com'
|
16
|
+
s.homepage = 'https://github.com/octplane/mongo-ec2-consistent-backup'
|
17
|
+
|
18
|
+
# These dependencies are only for people who work on this gem
|
19
|
+
s.add_dependency 'fog'
|
20
|
+
s.add_dependency 'bson_ext'
|
21
|
+
s.add_dependency 'trollop'
|
22
|
+
s.add_dependency 'mongo'
|
23
|
+
|
24
|
+
# Include everything in the lib folder
|
25
|
+
s.files = FileList['lib/**/*.rb', 'bin/*', '[A-Z]*', 'test/**/*'].to_a
|
26
|
+
|
27
|
+
s.executables << "mongo_lock_and_snapshot"
|
28
|
+
|
29
|
+
# Supress the warning about no rubyforge project
|
30
|
+
s.rubyforge_project = 'nowarning'
|
31
|
+
end
|
32
|
+
|
33
|
+
Rake::GemPackageTask.new(spec) do |package|
|
34
|
+
package.gem_spec = spec
|
35
|
+
# package.need_tar = true
|
36
|
+
# package.need_zip = true
|
37
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# Lock a set of disk via the mongo lock command and snapshot them to the cloud
|
3
|
+
|
4
|
+
require 'rubygems'
|
5
|
+
ENV['BUNDLE_GEMFILE'] = File.join(File.dirname(__FILE__), "..", "Gemfile")
|
6
|
+
require 'bundler/setup'
|
7
|
+
require 'trollop'
|
8
|
+
|
9
|
+
$: << File.join("..", File.dirname(__FILE__), "lib")
|
10
|
+
require 'ec2-consistent-backup'
|
11
|
+
require 'ec2_volume_snapshoter'
|
12
|
+
|
13
|
+
opts = Trollop::options do
|
14
|
+
opt :path, "Data path to freeze", :type => :string, :required => true
|
15
|
+
opt :access_key_id, "Access Key Id for AWS", :type => :string, :required => true
|
16
|
+
opt :secret_access_key, "Secret Access Key for AWS", :type => :string, :required => true
|
17
|
+
opt :devices, "Devices to snapshot, comma separated", :type => :string, :required => true
|
18
|
+
opt :type, "Snapshot type, to choose among #{EC2VolumeSnapshoter::KINDS.keys.join(",")}", :default => "snapshot"
|
19
|
+
opt :limit, "Cleanup old snapshots to keep only limit snapshots", :type => :integer
|
20
|
+
end
|
21
|
+
|
22
|
+
# find instance id by
|
23
|
+
# - resolving name to ip
|
24
|
+
# - looking in EC2 for server
|
25
|
+
# Lock Mongo
|
26
|
+
# Snapshot
|
27
|
+
# Unlock
|
28
|
+
|
29
|
+
aki = opts[:access_key_id]
|
30
|
+
sak = opts[:secret_access_key]
|
31
|
+
path = opts[:path]
|
32
|
+
|
33
|
+
`/usr/sbin/xfs_freeze -f #{path}`
|
34
|
+
|
35
|
+
begin
|
36
|
+
snapshoter = EC2VolumeSnapshoter.new(aki, sak)
|
37
|
+
limit = if opts[:limit] == nil
|
38
|
+
EC2VolumeSnapshoter::KINDS[opts[:type]]
|
39
|
+
else
|
40
|
+
opts[:limit]
|
41
|
+
end
|
42
|
+
|
43
|
+
snapshoter.snapshot_devices(opts[:devices].split(/,/), "Mongo Snapshot", opts[:type], limit)
|
44
|
+
rescue Exception => e
|
45
|
+
require "pp"
|
46
|
+
puts e.inspect
|
47
|
+
pp e.backtrace
|
48
|
+
ensure
|
49
|
+
`/usr/sbin/xfs_freeze -u #{path}`
|
50
|
+
end
|
@@ -0,0 +1,169 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'mongo'
|
3
|
+
require 'open-uri'
|
4
|
+
|
5
|
+
=begin
|
6
|
+
- check S3 credentials
|
7
|
+
- check disk location of data
|
8
|
+
- check this is on a remotely mounted disk (or md drive)
|
9
|
+
- http://www.mongodb.org/display/DOCS/getCmdLineOpts+command
|
10
|
+
=end
|
11
|
+
|
12
|
+
|
13
|
+
=begin
|
14
|
+
/proc/mdstat content:
|
15
|
+
|
16
|
+
Personalities : [raid0]
|
17
|
+
md0 : active raid0 sdo[3] sdn[2] sdm[1] sdl[0]
|
18
|
+
838859776 blocks 256k chunks
|
19
|
+
|
20
|
+
unused devices: <none>
|
21
|
+
=end
|
22
|
+
class NoSuchSetException < Exception; end
|
23
|
+
# Parse the existing RAID sets by reading /prod/mdstat
|
24
|
+
# Cheap alternative to using FFI to interface with libdm
|
25
|
+
class MDInspector
|
26
|
+
MDFILE = "/proc/mdstat"
|
27
|
+
PERSONALITIES = "Personalities :"
|
28
|
+
attr_reader :has_md, :personalities
|
29
|
+
attr_reader :drives
|
30
|
+
def initialize(mdfile = MDFILE)
|
31
|
+
@has_md = false
|
32
|
+
if File.exists?(mdfile)
|
33
|
+
stat_data = File.open(mdfile).read.split(/\n/)
|
34
|
+
personalities_line = stat_data.grep(/#{PERSONALITIES}/)
|
35
|
+
if personalities_line =~ /#{PERSONALITIES}(.+)/
|
36
|
+
@personalities = $1
|
37
|
+
else
|
38
|
+
@has_md = false
|
39
|
+
end
|
40
|
+
@set_metadata = {}
|
41
|
+
stat_data.grep(/^md[0-9]+ : /).each do |md_info|
|
42
|
+
if md_info =~ /^md([0-9]+) : active ([^ ]+) (.*)$/
|
43
|
+
set_name = "md#{$1}"
|
44
|
+
personality = $2
|
45
|
+
drives = $3.split(/ /).map{ |i| "/dev/"+i.gsub(/\[[0-9]+\]/,'') }.to_a
|
46
|
+
@set_metadata[set_name] = { :set_name=> set_name, :personality => personality,
|
47
|
+
:drives => drives}
|
48
|
+
end
|
49
|
+
end
|
50
|
+
@has_md = true if @set_metadata.keys.length > 0
|
51
|
+
end
|
52
|
+
end
|
53
|
+
# Returns the information about the MD set @name
|
54
|
+
def set(name)
|
55
|
+
# Handle "/dev/foobar" instead of "foobar"
|
56
|
+
if name =~ /\/dev\/(.*)$/
|
57
|
+
name = $1
|
58
|
+
end
|
59
|
+
return @set_metadata[name] if @set_metadata.has_key?(name)
|
60
|
+
raise NoSuchSetException.new(name)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
class NotMountedException < Exception; end
|
65
|
+
class MountInspector
|
66
|
+
def initialize(file = '/etc/mtab')
|
67
|
+
@dev_to_fs = {}
|
68
|
+
@fs_to_dev = {}
|
69
|
+
File.open(file).read.split(/\n/).map {|line| line.split(/ /)[0..1]}.each do |m|
|
70
|
+
@dev_to_fs[m[0]] = m[1] if m[0] != "none"
|
71
|
+
@fs_to_dev[m[1]] = m[0] if m[1] != "none"
|
72
|
+
end
|
73
|
+
end
|
74
|
+
def where_is_mounted(device)
|
75
|
+
return @dev_to_fs[device] if @dev_to_fs.has_key?(device)
|
76
|
+
raise NotMountedException.new(device)
|
77
|
+
end
|
78
|
+
def which_device(folder)
|
79
|
+
# Level 0 optimisation+ Handle "/" folder
|
80
|
+
return @fs_to_dev[folder] if @fs_to_dev.has_key?(folder)
|
81
|
+
|
82
|
+
components = folder.split(/\//)
|
83
|
+
components.size.downto(0).each do |sz|
|
84
|
+
current_folder = components[0..sz-1].join("/")
|
85
|
+
current_folder = "/" if current_folder == ""
|
86
|
+
return @fs_to_dev[current_folder] if @fs_to_dev.has_key?(current_folder)
|
87
|
+
end
|
88
|
+
raise NotMountedException.new(folder)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
module MongoHelper
|
93
|
+
class DataLocker
|
94
|
+
attr_reader :path
|
95
|
+
def initialize(port = 27017, host = 'localhost')
|
96
|
+
@m = Mongo::Connection.new(host, port)
|
97
|
+
args = @m['admin'].command({'getCmdLineOpts' => 1 })['argv']
|
98
|
+
p = args.index('--dbpath')
|
99
|
+
@path = args[p+1]
|
100
|
+
@path = File.readlink(@path) if File.symlink?(@path)
|
101
|
+
|
102
|
+
end
|
103
|
+
def lock
|
104
|
+
return if locked?
|
105
|
+
@m.lock!
|
106
|
+
while !locked? do
|
107
|
+
sleep(1)
|
108
|
+
end
|
109
|
+
raise "Not locked as asked" if !locked?
|
110
|
+
end
|
111
|
+
def locked?
|
112
|
+
@m.locked?
|
113
|
+
end
|
114
|
+
def unlock
|
115
|
+
return if !locked?
|
116
|
+
raise "Already unlocked" if !locked?
|
117
|
+
@m.unlock!
|
118
|
+
while locked? do
|
119
|
+
sleep(1)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
def log s
|
126
|
+
$stderr.puts "[#{Time.now}]: #{s}"
|
127
|
+
end
|
128
|
+
|
129
|
+
if __FILE__ == $0
|
130
|
+
require 'trollop'
|
131
|
+
opts = Trollop::options do
|
132
|
+
opt :port, "Mongo port to connect to", :default => 27017
|
133
|
+
end
|
134
|
+
|
135
|
+
# First connect to mongo and find the dbpath
|
136
|
+
port = opts[:port]
|
137
|
+
m = MongoHelper::DataLocker.new(port)
|
138
|
+
data_location = m.path
|
139
|
+
log "Mongo at #{port} has its data in #{data_location}."
|
140
|
+
|
141
|
+
|
142
|
+
mount_inspector = MountInspector.new
|
143
|
+
raid_set = mount_inspector.which_device(data_location)
|
144
|
+
log "This path is on the device #{raid_set}."
|
145
|
+
|
146
|
+
begin
|
147
|
+
raid_sets = MDInspector.new
|
148
|
+
drives = raid_sets.set(raid_set)[:drives]
|
149
|
+
|
150
|
+
log "This device is the MD device built with #{drives.inspect}."
|
151
|
+
|
152
|
+
# # this code probably works, but is way to dangerous.
|
153
|
+
# m.lock
|
154
|
+
# begin
|
155
|
+
# log "Locked mongo"
|
156
|
+
# e = EC2VolumeSnapshoter.new(opts[:access_key_id], opts[:secret_access_key], opts[:region])
|
157
|
+
# e.snapshot_devices(drives)
|
158
|
+
# rescue Exception => e
|
159
|
+
# puts e.inspect
|
160
|
+
# ensure
|
161
|
+
# m.unlock
|
162
|
+
# log "Unlocked mongo"
|
163
|
+
# end
|
164
|
+
|
165
|
+
rescue NoSuchSetException => e
|
166
|
+
log "Device #{raid_set} is not a MD device, bailing out"
|
167
|
+
raise e
|
168
|
+
end
|
169
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'resolv'
|
2
|
+
require 'fog'
|
3
|
+
|
4
|
+
class EC2InstanceNotFoundException < Exception; end
|
5
|
+
# Fetch an instance from its private ip address
|
6
|
+
class EC2InstanceIdentifier
|
7
|
+
# Need access_key_id, secret_access_key
|
8
|
+
def initialize(aki, sak)
|
9
|
+
@compute = Fog::Compute.new({:provider => 'AWS', :aws_access_key_id => aki, :aws_secret_access_key => sak})
|
10
|
+
end
|
11
|
+
# Returns the instance corresponding to the provided hostname
|
12
|
+
def get_instance(hostname)
|
13
|
+
ip = Resolv.getaddress(hostname)
|
14
|
+
instance = @compute.servers.all().find { |i| i.private_ip_address == ip || i.public_ip_address == ip }
|
15
|
+
raise InstanceNotFoundException.new(hostname) if ! instance
|
16
|
+
return instance
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
if __FILE__ == $0
|
21
|
+
require 'trollop'
|
22
|
+
require 'pp'
|
23
|
+
opts = Trollop::options do
|
24
|
+
opt :access_key_id, "Access Key Id for AWS", :type => :string, :required => true
|
25
|
+
opt :secret_access_key, "Secret Access Key for AWS", :type => :string, :required => true
|
26
|
+
opt :hostname, "Hostname to look for. Should resolve to a local EC2 Ip", :type => :string, :required => true
|
27
|
+
end
|
28
|
+
|
29
|
+
eii = EC2InstanceIdentifier.new(opts[:access_key_id], opts[:secret_access_key])
|
30
|
+
pp eii.get_instance(opts[:hostname])
|
31
|
+
|
32
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'fog'
|
2
|
+
require 'open-uri'
|
3
|
+
|
4
|
+
|
5
|
+
#
|
6
|
+
class SnapshotRestorer
|
7
|
+
def initialize(aki, sak, snap_ids)
|
8
|
+
@compute = Fog::Compute.new({:provider => 'AWS', :aws_access_key_id => aki, :aws_secret_access_key => sak})
|
9
|
+
@snaps = snap_ids
|
10
|
+
@volumes = []
|
11
|
+
end
|
12
|
+
def restore()
|
13
|
+
@snaps.each do | resource_id |
|
14
|
+
snap = @compute.snapshots.get(resource_id)
|
15
|
+
# Snap have the following tags
|
16
|
+
# application
|
17
|
+
# device
|
18
|
+
# instance_id
|
19
|
+
# date
|
20
|
+
# kind
|
21
|
+
|
22
|
+
t = snap.tags
|
23
|
+
volume = @compute.volumes.new :snapshot_id => snap.id, :size => snap.volume_size, :availability_zone => 'us-east-1c'
|
24
|
+
@compute.tags.create(:resource_id => volume.id, :key =>"application", :value => NAME_PREFIX)
|
25
|
+
@compute.tags.create(:resource_id => volume.id, :key =>"device", :value => device)
|
26
|
+
@compute.tags.create(:resource_id => volume.id, :key =>"date", :value => ts)
|
27
|
+
@compute.tags.create(:resource_id => volume.id, :key =>"kind", :value => kind)
|
28
|
+
volume.save
|
29
|
+
@volumes << volume
|
30
|
+
end
|
31
|
+
end
|
32
|
+
def connect(instance_id = open("http://169.254.169.254/latest/meta-data/instance-id").read)
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,169 @@
|
|
1
|
+
require 'fog'
|
2
|
+
require 'open-uri'
|
3
|
+
|
4
|
+
# This class is responsible of the snapshoting of given disks to EC2
|
5
|
+
# EC2 related permissions in IAM
|
6
|
+
# Sid": "Stmt1344254048404",
|
7
|
+
# "Action": [
|
8
|
+
# "ec2:CreateSnapshot",
|
9
|
+
# "ec2:DeleteSnapshot",
|
10
|
+
# "ec2:DescribeSnapshots",
|
11
|
+
# "ec2:CreateTags",
|
12
|
+
# "ec2:DescribeTags",
|
13
|
+
# "ec2:DescribeVolumes"
|
14
|
+
# ],
|
15
|
+
# "Effect": "Allow",
|
16
|
+
# "Resource": [
|
17
|
+
# "*"
|
18
|
+
# ]
|
19
|
+
#
|
20
|
+
|
21
|
+
class NoSuchVolumeException < Exception
|
22
|
+
def initialize(instance, volume, details)
|
23
|
+
@instance, @volume, @details = instance, volume, details
|
24
|
+
end
|
25
|
+
def to_s
|
26
|
+
"Unable to locate volume \"#{@volume}\" on #{@instance}\nKnow volumes for this instance are:\n#{@details.inspect}"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def log s
|
31
|
+
$stderr.puts "[#{Time.now}]: #{s}"
|
32
|
+
end
|
33
|
+
|
34
|
+
class EC2VolumeSnapshoter
|
35
|
+
NAME_PREFIX='Volume Snapshot'
|
36
|
+
|
37
|
+
# Kind of snapshot and their expiration in days
|
38
|
+
KINDS = { 'test' => 1,
|
39
|
+
'snapshot' => 0,
|
40
|
+
'daily' => 7,
|
41
|
+
'weekly' => 31,
|
42
|
+
'monthly' => 300,
|
43
|
+
'yearly' => 0}
|
44
|
+
|
45
|
+
attr_reader :instance_id
|
46
|
+
# Need access_key_id, secret_access_key and instance_id
|
47
|
+
# If not provided, attempt to fetch current instance_id
|
48
|
+
def initialize(aki, sak, instance_id = open("http://169.254.169.254/latest/meta-data/instance-id").read)
|
49
|
+
|
50
|
+
@instance_id = instance_id
|
51
|
+
|
52
|
+
@compute = Fog::Compute.new({:provider => 'AWS', :aws_access_key_id => aki, :aws_secret_access_key => sak})
|
53
|
+
end
|
54
|
+
# Snapshots the list of devices
|
55
|
+
# devices is an array of device attached to the instance (/dev/foo)
|
56
|
+
# name if the name of the snapshot
|
57
|
+
def snapshot_devices(devices, name = "#{instance_id}", kind = "test", limit = KINDS[kind])
|
58
|
+
log "Snapshot of kind #{kind}, limit set to #{limit} (0 means never purge)"
|
59
|
+
ts = DateTime.now.to_s
|
60
|
+
name = "#{NAME_PREFIX}:" + name
|
61
|
+
volumes = {}
|
62
|
+
devices.each do |device|
|
63
|
+
volumes[device] = find_volume_for_device(device)
|
64
|
+
end
|
65
|
+
sn = []
|
66
|
+
volumes.each do |device, volume|
|
67
|
+
log "Creating volume snapshot for #{device} on instance #{instance_id}"
|
68
|
+
snapshot = volume.snapshots.new
|
69
|
+
snapshot.description = name+" #{device}"
|
70
|
+
snapshot.save
|
71
|
+
sn << snapshot
|
72
|
+
snapshot.reload
|
73
|
+
|
74
|
+
@compute.tags.create(:resource_id => snapshot.id, :key =>"application", :value => NAME_PREFIX)
|
75
|
+
@compute.tags.create(:resource_id => snapshot.id, :key =>"device", :value => device)
|
76
|
+
@compute.tags.create(:resource_id => snapshot.id, :key =>"instance_id", :value =>instance_id)
|
77
|
+
@compute.tags.create(:resource_id => snapshot.id, :key =>"date", :value => ts)
|
78
|
+
@compute.tags.create(:resource_id => snapshot.id, :key =>"kind", :value => kind)
|
79
|
+
|
80
|
+
end
|
81
|
+
log "Waiting for snapshots to complete."
|
82
|
+
sn.each do |s|
|
83
|
+
begin
|
84
|
+
sleep(3)
|
85
|
+
s.reload
|
86
|
+
end while s.state == 'nil' || s.state == 'pending'
|
87
|
+
end
|
88
|
+
|
89
|
+
if limit != 0
|
90
|
+
# populate data structure with updated information
|
91
|
+
snapshots = list_snapshots(devices, kind)
|
92
|
+
nsnaps = snapshots.keys.length
|
93
|
+
if nsnaps-limit > 0
|
94
|
+
dates = snapshots.keys.sort
|
95
|
+
puts dates.inspect
|
96
|
+
extra_snapshots = dates[0..-limit]
|
97
|
+
remaining_snapshots = dates[-limit..-1]
|
98
|
+
extra_snapshots.each do |date|
|
99
|
+
snapshots[date].each do |snap|
|
100
|
+
log "Destroying #{snap.description} #{snap.id}"
|
101
|
+
snap.destroy
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
# List snapshots for a set of device and a given kind
|
109
|
+
require 'pp'
|
110
|
+
def list_snapshots(devices, kind)
|
111
|
+
volume_map = []
|
112
|
+
snapshots = {}
|
113
|
+
|
114
|
+
tags = @compute.tags.all(:key => 'instance_id', :value => instance_id)
|
115
|
+
tags.each do |tag|
|
116
|
+
snap = @compute.snapshots.get(tag.resource_id)
|
117
|
+
t = snap.tags
|
118
|
+
|
119
|
+
if devices.include?(t['device']) &&
|
120
|
+
instance_id == t['instance_id'] &&
|
121
|
+
NAME_PREFIX == t['application'] &&
|
122
|
+
kind == t['kind']
|
123
|
+
snapshots[t['date']] ||= []
|
124
|
+
snapshots[t['date']] << snap
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
# take out incomplete backups
|
129
|
+
snapshots.delete_if{ |date, snaps| snaps.length != devices.length }
|
130
|
+
snapshots
|
131
|
+
end
|
132
|
+
|
133
|
+
|
134
|
+
def find_volume_for_device(device)
|
135
|
+
my = []
|
136
|
+
@compute.volumes.all().each do |volume|
|
137
|
+
if volume.server_id == @instance_id
|
138
|
+
my << volume
|
139
|
+
if volume.device == device
|
140
|
+
return volume
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
raise NoSuchVolumeException.new(@instance_id, device, my)
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
if __FILE__ == $0
|
149
|
+
require 'trollop'
|
150
|
+
require 'pp'
|
151
|
+
|
152
|
+
opts = Trollop::options do
|
153
|
+
opt :access_key_id, "Access Key Id for AWS", :type => :string, :required => true
|
154
|
+
opt :secret_access_key, "Secret Access Key for AWS", :type => :string, :required => true
|
155
|
+
opt :instance_id, "Instance identifier", :type => :string, :required => true
|
156
|
+
opt :find_volume_for, "Show information for device path (mount point)", :type => :string
|
157
|
+
opt :snapshot, "Snapshot device path (mount point)", :type => :string
|
158
|
+
opt :snapshot_type, "Kind of snapshot (any of #{EC2VolumeSnapshoter::KINDS.keys.join(", ")})", :default => 'test'
|
159
|
+
|
160
|
+
end
|
161
|
+
|
162
|
+
evs = EC2VolumeSnapshoter.new(opts[:access_key_id], opts[:secret_access_key], opts[:instance_id])
|
163
|
+
if opts[:find_volume_for]
|
164
|
+
pp evs.find_volume_for_device(opts[:find_volume_for])
|
165
|
+
end
|
166
|
+
if opts[:snapshot]
|
167
|
+
evs.snapshot_devices([opts[:snapshot]])
|
168
|
+
end
|
169
|
+
end
|
metadata
ADDED
@@ -0,0 +1,119 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: mongo-ec2-backup
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.4
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Pierre Baillet
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-08-06 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fog
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: bson_ext
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: trollop
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: mongo
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :runtime
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
description:
|
79
|
+
email: oct@fotopedia.com
|
80
|
+
executables:
|
81
|
+
- mongo_lock_and_snapshot
|
82
|
+
extensions: []
|
83
|
+
extra_rdoc_files: []
|
84
|
+
files:
|
85
|
+
- lib/ec2-consistent-backup.rb
|
86
|
+
- lib/ec2_instance_identifier.rb
|
87
|
+
- lib/ec2_snapshot_restorer.rb
|
88
|
+
- lib/ec2_volume_snapshoter.rb
|
89
|
+
- bin/mongo_lock_and_snapshot
|
90
|
+
- Gemfile
|
91
|
+
- Gemfile.lock
|
92
|
+
- LICENCE
|
93
|
+
- Rakefile
|
94
|
+
- README.markdown
|
95
|
+
homepage: https://github.com/octplane/mongo-ec2-consistent-backup
|
96
|
+
licenses: []
|
97
|
+
post_install_message:
|
98
|
+
rdoc_options: []
|
99
|
+
require_paths:
|
100
|
+
- lib
|
101
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
102
|
+
none: false
|
103
|
+
requirements:
|
104
|
+
- - ! '>='
|
105
|
+
- !ruby/object:Gem::Version
|
106
|
+
version: '0'
|
107
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
108
|
+
none: false
|
109
|
+
requirements:
|
110
|
+
- - ! '>='
|
111
|
+
- !ruby/object:Gem::Version
|
112
|
+
version: '0'
|
113
|
+
requirements: []
|
114
|
+
rubyforge_project: nowarning
|
115
|
+
rubygems_version: 1.8.23
|
116
|
+
signing_key:
|
117
|
+
specification_version: 3
|
118
|
+
summary: Snapshot your mongodb in the EC2 cloud via XFS Freeze
|
119
|
+
test_files: []
|