mondupe 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/bin/mondupe +152 -0
- data/lib/mondupe.rb +142 -0
- metadata +62 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 34fd78b237de1808b0864a03c5849ec4502a775e
|
4
|
+
data.tar.gz: a07ddd4d9f5bc36c9d2495158cb2661c8b460434
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 3ebc59372c5391aa062007d073a8742719077a59a6084094d951fe526b01b90501a25b33e3f119fb2032710d7f108141f3988af7b86a0fe42556e25bc8ad2c4b
|
7
|
+
data.tar.gz: 455bb2195c14051966791fb1a15effb331ecd10fdab9c55a2a1bbb6fe82058a45d5e3880e02c9da285851ff22ccf2abc6b701a98e4e33ca75b7c6ebf43fdafe4
|
data/bin/mondupe
ADDED
@@ -0,0 +1,152 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'mondupe'
|
4
|
+
require 'optparse'
|
5
|
+
|
6
|
+
$options = {}
|
7
|
+
|
8
|
+
ValidHostnameRegex=/(?!-)[A-Z\d-]{1,63}(?<!-)$/
|
9
|
+
|
10
|
+
opt_parser = OptionParser.new do |opt|
|
11
|
+
opt.banner = "Usage: mondupe COMMAND [OPTIONS]"
|
12
|
+
opt.separator ""
|
13
|
+
opt.separator "Commands Description"
|
14
|
+
opt.separator " create - create new mongo host."
|
15
|
+
opt.separator " Requires -n [name]"
|
16
|
+
opt.separator " Optional -i [image] -e [expire_days] -o [owner] -t [type] -a [ipaddress] -s [size]"
|
17
|
+
opt.separator " delete - delete existing mongo host."
|
18
|
+
opt.separator " Requires -n [name] || -d [id]"
|
19
|
+
opt.separator " bootstrap - push chef-client and configuration to the node."
|
20
|
+
opt.separator " Requires -n [name] -a [ipaddress]"
|
21
|
+
opt.separator " dumps3 - retrieve the database dump from s3."
|
22
|
+
opt.separator " Requires -n [name] -a [ipaddress]"
|
23
|
+
opt.separator " restore - Restore a mongo dump that exists on a host"
|
24
|
+
opt.separator " Requires -n [name] -a [ipaddress]"
|
25
|
+
opt.separator " Optional -t [tmpdir]"
|
26
|
+
opt.separator " expire - (coming soon) Reset the expiration days of a node"
|
27
|
+
opt.separator " Requires ( -n [name] || -d [id] || -a [ipaddress] ) -e [expire_days]"
|
28
|
+
opt.separator " list - (coming soon) List all mongo hosts."
|
29
|
+
opt.separator " help - Get help."
|
30
|
+
opt.separator ""
|
31
|
+
opt.separator "Options"
|
32
|
+
|
33
|
+
opt.on("-n","--name HOSTNAME","Name of the host that you are creating") do |name|
|
34
|
+
if ValidHostnameRegex.match(name)
|
35
|
+
$options[:name] = name
|
36
|
+
else
|
37
|
+
abort("Invalid hostname")
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
opt.on("-i","--image IMAGENAME","Name of the AWS Image you would like to use") do |image|
|
42
|
+
$options[:image] = image
|
43
|
+
end
|
44
|
+
|
45
|
+
opt.on("-e","--expire DAYS","Number of days the instance should stay on line. - Default: 3") do |days|
|
46
|
+
$options[:expire_days] = days
|
47
|
+
end
|
48
|
+
|
49
|
+
opt.on("-o","--owner NAME","Name of the owner of the host to be created. - Default: Your Username") do |owner|
|
50
|
+
$options[:owner] = owner
|
51
|
+
end
|
52
|
+
|
53
|
+
opt.on("-t","--type INSTANCE_TYPE","Type of AWS host to create. - Default: m3.xlarge") do |type|
|
54
|
+
$options[:type] = type
|
55
|
+
end
|
56
|
+
|
57
|
+
opt.on("-a","--ipaddress IPADDRESS","IP address of node that already exists. Not to be used with creating new nodes") do |address|
|
58
|
+
$options[:ipaddress] = address
|
59
|
+
end
|
60
|
+
|
61
|
+
opt.on("-d","--id INSTANCE_ID","AWS ID of the instance, must exist already") do |id|
|
62
|
+
$options[:id] = id
|
63
|
+
end
|
64
|
+
|
65
|
+
opt.on("-t","--tmpdir TEMP_DIR","Directory on remote host that will be used for downloading the mongo dump. - Default: /tmp") do |tmpdir|
|
66
|
+
$options[:tmpdir] = tmpdir
|
67
|
+
end
|
68
|
+
|
69
|
+
opt.on("-s","--size VOLUME_SIZE","Set the root volume size in GB - Default: 60") do |size|
|
70
|
+
$options[:size] = size
|
71
|
+
end
|
72
|
+
|
73
|
+
opt.on("-h","--help","help") do
|
74
|
+
puts opt_parser
|
75
|
+
end
|
76
|
+
|
77
|
+
opt.separator "Environment Variables - Set these in your environment. They should be self explanitory."
|
78
|
+
opt.separator " Required:"
|
79
|
+
opt.separator " - MONDUPE_ROUTE53_DOMAIN"
|
80
|
+
opt.separator " - MONDUPE_SECURITY_GROUP"
|
81
|
+
opt.separator " - MONDUPE_S3_BUCKET_NAME"
|
82
|
+
opt.separator " - MONDUPE_KEY_PAIR_NAME"
|
83
|
+
opt.separator ""
|
84
|
+
opt.separator " Optional: (have sane defaults)"
|
85
|
+
opt.separator " - MONDUPE_INSTANCE_IMAGE_ID"
|
86
|
+
opt.separator " - MONDUPE_CHEF_RUN_LIST"
|
87
|
+
opt.separator " - MONDUPE_CHEF_IDENTITY_FILE"
|
88
|
+
opt.separator " - MONDUPE_CHEF_ENVIRONMENT"
|
89
|
+
opt.separator " - MONDUPE_SSH_KEY"
|
90
|
+
opt.separator " - MONDUPE_SSH_USER"
|
91
|
+
opt.separator " - MONDUPE_DUMP_FILE_NAME"
|
92
|
+
end
|
93
|
+
|
94
|
+
opt_parser.parse!
|
95
|
+
|
96
|
+
instance_image_id = $options[:image] || ENV['MONDUPE_INSTANCE_IMAGE_ID'] || "ami-018c9568"
|
97
|
+
instance_type = $options[:type] || "m3.xlarge"
|
98
|
+
instance_name = $options[:name] || nil
|
99
|
+
instance_owner = $options[:owner] || ENV['USER'] || 'mondupe'
|
100
|
+
instance_ipaddress = $options[:ipaddress] || nil
|
101
|
+
instance_id = $options[:id] || nil
|
102
|
+
dump_tmp_path = $options[:tmpdir] || '/tmp'
|
103
|
+
expire_days = $options[:expire_days] || 3
|
104
|
+
instance_count = 1
|
105
|
+
chef_run_list = ENV['MONDUPE_CHEF_RUN_LIST'] || ""
|
106
|
+
chef_environment = ENV['MONDUPE_CHEF_ENVIRONMENT'] || "default"
|
107
|
+
ssh_key = ENV['MONDUPE_SSH_KEY'] || "~/.ssh/id_rsa"
|
108
|
+
chef_identity_file = ENV['MONDUPE_CHEF_IDENTITY_FILE'] || ssh_key
|
109
|
+
ssh_user = ENV['MONDUPE_SSH_USER'] || "ubuntu"
|
110
|
+
route53_domain = ENV['MONDUPE_ROUTE53_DOMAIN'] || nil
|
111
|
+
instance_fqdn = ( instance_name + "." + route53_domain ) unless instance_name.nil? || route53_domain.nil?
|
112
|
+
key_pair_name = ENV['MONDUPE_KEY_PAIR_NAME'] || nil
|
113
|
+
security_group = ENV['MONDUPE_SECURITY_GROUP'] || nil
|
114
|
+
s3_bucket_name = ENV['MONDUPE_S3_BUCKET_NAME'] || nil
|
115
|
+
dump_file_name = ENV['MONDUPE_DUMP_FILE_NAME'] || 'mongodb.dump.tgz'
|
116
|
+
instance_volume_size = 60
|
117
|
+
|
118
|
+
case ARGV[0]
|
119
|
+
when "create"
|
120
|
+
start_time = Time.now
|
121
|
+
total_seconds = Time.now - start_time
|
122
|
+
puts "Creating AWS EC2 Instance with MongoDB and restoring from latest production backup"
|
123
|
+
instance = Mondupe.new.create_instance(instance_name, instance_image_id, instance_type, instance_count, security_group, key_pair_name, expire_days, instance_owner, instance_volume_size)
|
124
|
+
Mondupe.new.create_dns(instance_fqdn, route53_domain, instance)
|
125
|
+
Mondupe.new.bootstrap(instance_name, instance_fqdn, instance.ip_address, chef_environment, chef_identity_file, chef_run_list, ssh_user)
|
126
|
+
Mondupe.new.get_db_dump_from_s3(instance.ip_address, s3_bucket_name, dump_tmp_path, ssh_user, dump_file_name)
|
127
|
+
Mondupe.new.restore_db(instance.ip_address, dump_tmp_path, ssh_key, ssh_user, dump_file_name)
|
128
|
+
puts " - - - Total Run Time: #{((total_seconds % 3600) / 60).to_i}m #{((total_seconds % 3600) % 60).to_i}s - - -"
|
129
|
+
when "delete"
|
130
|
+
puts "delete mongo host"
|
131
|
+
when "bootstrap"
|
132
|
+
puts "bootstrapping node"
|
133
|
+
# Find the instance and create the instance object here
|
134
|
+
Mondupe.new.bootstrap(instance_name, instance_fqdn, instance_ipaddress, chef_environment, chef_identity_file, chef_run_list, ssh_user)
|
135
|
+
when "dumps3"
|
136
|
+
puts "getting dump"
|
137
|
+
Mondupe.new.get_db_dump_from_s3(instance_ipaddress, s3_bucket_name, dump_tmp_path, ssh_user, dump_file_name)
|
138
|
+
when "restore"
|
139
|
+
puts "Restoring mongo database from dump"
|
140
|
+
Mondupe.new.restore_db(instance_ipaddress, dump_tmp_path, ssh_key, ssh_user, dump_file_name)
|
141
|
+
when "terminate"
|
142
|
+
puts "Marking instance for termination"
|
143
|
+
Mondupe.new.terminate_instance(instance_id)
|
144
|
+
when "list"
|
145
|
+
puts "Listing all instances created by MonDupe"
|
146
|
+
Mondupe.new.list_instances
|
147
|
+
when "expire"
|
148
|
+
puts "Modifying expiration days for instance"
|
149
|
+
Mondupe.new.instance_expire(instance_id, instance_name, expire_days)
|
150
|
+
else
|
151
|
+
puts opt_parser
|
152
|
+
end
|
data/lib/mondupe.rb
ADDED
@@ -0,0 +1,142 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'aws-sdk'
|
4
|
+
|
5
|
+
class Mondupe
|
6
|
+
def create_instance(instance_name, instance_image_id, instance_type, instance_count, security_group, key_pair_name, expire_days, instance_owner, instance_volume_size)
|
7
|
+
AWS.config(:access_key_id => ENV['AWS_ACCESS_KEY_ID'], :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'], region: 'us-east-1')
|
8
|
+
|
9
|
+
ec2 = AWS::EC2.new(:access_key_id => ENV['AWS_ACCESS_KEY_ID'], :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'])
|
10
|
+
|
11
|
+
puts "Creating with - Security Group: #{security_group} Instance Owner: #{instance_owner}"
|
12
|
+
|
13
|
+
key_pair = ec2.key_pairs[key_pair_name]
|
14
|
+
|
15
|
+
# Use this to create a new security group - Can have preset options
|
16
|
+
#security_group = ec2.security_groups.create("sg_#{instance_name}")
|
17
|
+
#security_group = 'sg_my_awesome_new_instance'
|
18
|
+
|
19
|
+
instance = ec2.instances.create(
|
20
|
+
:image_id => instance_image_id,
|
21
|
+
:block_device_mappings => [{
|
22
|
+
:device_name => "/dev/sda1",
|
23
|
+
:ebs => {
|
24
|
+
:volume_size => instance_volume_size,
|
25
|
+
:delete_on_termination => true
|
26
|
+
}
|
27
|
+
}],
|
28
|
+
:instance_type => instance_type,
|
29
|
+
:count => instance_count,
|
30
|
+
:security_groups => security_group,
|
31
|
+
:key_pair => key_pair
|
32
|
+
)
|
33
|
+
|
34
|
+
created_date_time = Time.now.to_i
|
35
|
+
|
36
|
+
# Display some information about the new instance
|
37
|
+
puts "Instance '#{instance_name}' created with ID '#{instance.id}'"
|
38
|
+
|
39
|
+
instance.tag('Name', :value => instance_name)
|
40
|
+
instance.tag('owner', :value => instance_owner)
|
41
|
+
instance.tag('expire_days', :value => expire_days)
|
42
|
+
instance.tag('created', :value => created_date_time)
|
43
|
+
instance.tag('mondupe')
|
44
|
+
|
45
|
+
puts "Added tags... "
|
46
|
+
puts " Name: #{instance_name}"
|
47
|
+
puts " owner: #{instance_owner}"
|
48
|
+
puts " expires: #{expire_days}"
|
49
|
+
puts " created: #{created_date_time}"
|
50
|
+
|
51
|
+
# Wait for instance to be ready
|
52
|
+
current_state = ""
|
53
|
+
until instance.status == :running
|
54
|
+
if current_state != instance.status
|
55
|
+
puts "Status: #{instance.status.to_s}"
|
56
|
+
print "Instance coming up "
|
57
|
+
current_state = instance.status
|
58
|
+
else
|
59
|
+
print "."
|
60
|
+
sleep 1
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
puts ""
|
65
|
+
puts "Instance #{instance.id} is now running"
|
66
|
+
puts "Name: #{instance.tags['Name']}"
|
67
|
+
puts "IP: #{instance.ip_address}"
|
68
|
+
puts "Public DNS: #{instance.dns_name}"
|
69
|
+
instance
|
70
|
+
end
|
71
|
+
|
72
|
+
def create_dns(instance_fqdn, route53_domain, instance)
|
73
|
+
# Set up DNS through Route53
|
74
|
+
puts "Setting up Route53 DNS..."
|
75
|
+
# Check to see if record exists
|
76
|
+
route53 = AWS::Route53.new(:access_key_id => ENV['AWS_ACCESS_KEY_ID'], :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'])
|
77
|
+
zone = route53.hosted_zones.select { |z| z.name == route53_domain }.first
|
78
|
+
|
79
|
+
rrsets = AWS::Route53::HostedZone.new(zone.id).rrsets
|
80
|
+
rrset = rrsets.create(instance_fqdn, 'A', :ttl => 300, :resource_records => [{:value => instance.ip_address }])
|
81
|
+
if rrset.exists?
|
82
|
+
# Create new record if does not exist
|
83
|
+
rrset.update
|
84
|
+
else
|
85
|
+
# Update if record exists
|
86
|
+
rrset = zone.rrsets[instance_fqdn, 'A']
|
87
|
+
rrset.resource_records = [ { :value => instance.ip_address } ]
|
88
|
+
rrset.update
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
|
93
|
+
def bootstrap(instance_name, instance_fqdn, instance_ipaddress, chef_environment, chef_identity_file, chef_run_list, ssh_user)
|
94
|
+
# Bootstrap the new instance with chef
|
95
|
+
puts "Bootstraping node with Chef..."
|
96
|
+
puts "Running..."
|
97
|
+
puts "knife bootstrap #{instance_ipaddress} -N #{instance_fqdn[0...-1]} -E #{chef_environment} -i #{chef_identity_file} -r #{chef_run_list} -x #{ssh_user} --sudo"
|
98
|
+
sleep 30
|
99
|
+
system("knife bootstrap #{instance_ipaddress} -N #{instance_fqdn[0...-1]} -E #{chef_environment} -i #{chef_identity_file} -r #{chef_run_list} -x #{ssh_user} --sudo")
|
100
|
+
end
|
101
|
+
|
102
|
+
def get_db_dump_from_s3(instance_ip, s3_bucket_name, dump_tmp_path, ssh_user, dump_file_name)
|
103
|
+
expiration = Time.now.to_i + 400*60
|
104
|
+
s3 = AWS::S3.new(:access_key_id => ENV['AWS_ACCESS_KEY_ID'], :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'])
|
105
|
+
backups = s3.buckets[s3_bucket_name]
|
106
|
+
latest_backup = backups.objects.sort_by {|backup| backup.last_modified}.last
|
107
|
+
download_url = latest_backup.url_for(:get, :expires_in => expiration, :response_content_type => "application/json")
|
108
|
+
puts "Download URL: #{download_url}"
|
109
|
+
puts "#{Time.now.to_s} - Starting download."
|
110
|
+
puts " Please wait..."
|
111
|
+
`ssh -i ~/.ssh/DevOps.pem #{ssh_user}@#{instance_ip} "cd #{dump_tmp_path} && wget '#{download_url}' -O #{File.join(dump_tmp_path, dump_file_name)} 2&>1"`
|
112
|
+
puts "#{Time.now.to_s} - Download completed"
|
113
|
+
end
|
114
|
+
|
115
|
+
def restore_db(instance_ip, dump_tmp_path, ssh_key, ssh_user, dump_file_name)
|
116
|
+
# Restore from the database dump
|
117
|
+
# TODO - Fail the process if any step fails
|
118
|
+
puts "#{Time.now.to_s} - Dropping existing database"
|
119
|
+
`ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "echo 'db.dropDatabase()' | mongo cde_production"`
|
120
|
+
puts "#{Time.now.to_s} - Database drop complete"
|
121
|
+
puts "Restoring Mongo Database from extracted dump: #{File.join(dump_tmp_path, "cde_production")}"
|
122
|
+
`ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "cd /tmp; tar xf #{dump_file_name}; time mongorestore /tmp/cde_production"`
|
123
|
+
puts "Removing database archive file"
|
124
|
+
`ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "rm -rf /tmp/#{File.join(dump_tmp_path, dump_file_name)}"`
|
125
|
+
puts "#{Time.now.to_s} - Removing saved searches"
|
126
|
+
`ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "mongo cde_production --eval \\"db.users.update({save_searches: {$ne: null}}, {$unset: {save_searches: ''}}, {multi: true})\\""`
|
127
|
+
puts "#{Time.now.to_s} - Cleaning up our mess..."
|
128
|
+
`ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "rm -rf #{File.join(dump_tmp_path, 'cde_production')}"`
|
129
|
+
end
|
130
|
+
|
131
|
+
def terminate(instance_id)
|
132
|
+
puts "function not quite ready yet"
|
133
|
+
end
|
134
|
+
|
135
|
+
def list
|
136
|
+
puts "function not quite ready yet"
|
137
|
+
end
|
138
|
+
|
139
|
+
def expire(instance_id, instance_name, expire_days)
|
140
|
+
puts "function not quite ready yet"
|
141
|
+
end
|
142
|
+
end
|
metadata
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: mondupe
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Philip Hutchins
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-04-25 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.38'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.38'
|
27
|
+
description: Create an AWS EC2 node and restore a MongoDB dump to it from an AWS S3
|
28
|
+
bucket
|
29
|
+
email: flipture@gmail.com
|
30
|
+
executables:
|
31
|
+
- mondupe
|
32
|
+
extensions: []
|
33
|
+
extra_rdoc_files: []
|
34
|
+
files:
|
35
|
+
- bin/mondupe
|
36
|
+
- lib/mondupe.rb
|
37
|
+
homepage: http://phutchins.com
|
38
|
+
licenses:
|
39
|
+
- MIT
|
40
|
+
metadata: {}
|
41
|
+
post_install_message:
|
42
|
+
rdoc_options: []
|
43
|
+
require_paths:
|
44
|
+
- lib
|
45
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - '>='
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: '0'
|
50
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
requirements: []
|
56
|
+
rubyforge_project:
|
57
|
+
rubygems_version: 2.2.2
|
58
|
+
signing_key:
|
59
|
+
specification_version: 4
|
60
|
+
summary: MonDupe
|
61
|
+
test_files: []
|
62
|
+
has_rdoc:
|