mondupe 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. checksums.yaml +7 -0
  2. data/bin/mondupe +152 -0
  3. data/lib/mondupe.rb +142 -0
  4. metadata +62 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 34fd78b237de1808b0864a03c5849ec4502a775e
4
+ data.tar.gz: a07ddd4d9f5bc36c9d2495158cb2661c8b460434
5
+ SHA512:
6
+ metadata.gz: 3ebc59372c5391aa062007d073a8742719077a59a6084094d951fe526b01b90501a25b33e3f119fb2032710d7f108141f3988af7b86a0fe42556e25bc8ad2c4b
7
+ data.tar.gz: 455bb2195c14051966791fb1a15effb331ecd10fdab9c55a2a1bbb6fe82058a45d5e3880e02c9da285851ff22ccf2abc6b701a98e4e33ca75b7c6ebf43fdafe4
data/bin/mondupe ADDED
@@ -0,0 +1,152 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'mondupe'
4
+ require 'optparse'
5
+
6
+ $options = {}
7
+
8
+ ValidHostnameRegex=/(?!-)[A-Z\d-]{1,63}(?<!-)$/
9
+
10
+ opt_parser = OptionParser.new do |opt|
11
+ opt.banner = "Usage: mondupe COMMAND [OPTIONS]"
12
+ opt.separator ""
13
+ opt.separator "Commands Description"
14
+ opt.separator " create - create new mongo host."
15
+ opt.separator " Requires -n [name]"
16
+ opt.separator " Optional -i [image] -e [expire_days] -o [owner] -t [type] -a [ipaddress] -s [size]"
17
+ opt.separator " delete - delete existing mongo host."
18
+ opt.separator " Requires -n [name] || -d [id]"
19
+ opt.separator " bootstrap - push chef-client and configuration to the node."
20
+ opt.separator " Requires -n [name] -a [ipaddress]"
21
+ opt.separator " dumps3 - retrieve the database dump from s3."
22
+ opt.separator " Requires -n [name] -a [ipaddress]"
23
+ opt.separator " restore - Restore a mongo dump that exists on a host"
24
+ opt.separator " Requires -n [name] -a [ipaddress]"
25
+ opt.separator " Optional -t [tmpdir]"
26
+ opt.separator " expire - (coming soon) Reset the expiration days of a node"
27
+ opt.separator " Requires ( -n [name] || -d [id] || -a [ipaddress] ) -e [expire_days]"
28
+ opt.separator " list - (coming soon) List all mongo hosts."
29
+ opt.separator " help - Get help."
30
+ opt.separator ""
31
+ opt.separator "Options"
32
+
33
+ opt.on("-n","--name HOSTNAME","Name of the host that you are creating") do |name|
34
+ if ValidHostnameRegex.match(name)
35
+ $options[:name] = name
36
+ else
37
+ abort("Invalid hostname")
38
+ end
39
+ end
40
+
41
+ opt.on("-i","--image IMAGENAME","Name of the AWS Image you would like to use") do |image|
42
+ $options[:image] = image
43
+ end
44
+
45
+ opt.on("-e","--expire DAYS","Number of days the instance should stay on line. - Default: 3") do |days|
46
+ $options[:expire_days] = days
47
+ end
48
+
49
+ opt.on("-o","--owner NAME","Name of the owner of the host to be created. - Default: Your Username") do |owner|
50
+ $options[:owner] = owner
51
+ end
52
+
53
+ opt.on("-t","--type INSTANCE_TYPE","Type of AWS host to create. - Default: m3.xlarge") do |type|
54
+ $options[:type] = type
55
+ end
56
+
57
+ opt.on("-a","--ipaddress IPADDRESS","IP address of node that already exists. Not to be used with creating new nodes") do |address|
58
+ $options[:ipaddress] = address
59
+ end
60
+
61
+ opt.on("-d","--id INSTANCE_ID","AWS ID of the instance, must exist already") do |id|
62
+ $options[:id] = id
63
+ end
64
+
65
+ opt.on("-t","--tmpdir TEMP_DIR","Directory on remote host that will be used for downloading the mongo dump. - Default: /tmp") do |tmpdir|
66
+ $options[:tmpdir] = tmpdir
67
+ end
68
+
69
+ opt.on("-s","--size VOLUME_SIZE","Set the root volume size in GB - Default: 60") do |size|
70
+ $options[:size] = size
71
+ end
72
+
73
+ opt.on("-h","--help","help") do
74
+ puts opt_parser
75
+ end
76
+
77
+ opt.separator "Environment Variables - Set these in your environment. They should be self explanitory."
78
+ opt.separator " Required:"
79
+ opt.separator " - MONDUPE_ROUTE53_DOMAIN"
80
+ opt.separator " - MONDUPE_SECURITY_GROUP"
81
+ opt.separator " - MONDUPE_S3_BUCKET_NAME"
82
+ opt.separator " - MONDUPE_KEY_PAIR_NAME"
83
+ opt.separator ""
84
+ opt.separator " Optional: (have sane defaults)"
85
+ opt.separator " - MONDUPE_INSTANCE_IMAGE_ID"
86
+ opt.separator " - MONDUPE_CHEF_RUN_LIST"
87
+ opt.separator " - MONDUPE_CHEF_IDENTITY_FILE"
88
+ opt.separator " - MONDUPE_CHEF_ENVIRONMENT"
89
+ opt.separator " - MONDUPE_SSH_KEY"
90
+ opt.separator " - MONDUPE_SSH_USER"
91
+ opt.separator " - MONDUPE_DUMP_FILE_NAME"
92
+ end
93
+
94
+ opt_parser.parse!
95
+
96
+ instance_image_id = $options[:image] || ENV['MONDUPE_INSTANCE_IMAGE_ID'] || "ami-018c9568"
97
+ instance_type = $options[:type] || "m3.xlarge"
98
+ instance_name = $options[:name] || nil
99
+ instance_owner = $options[:owner] || ENV['USER'] || 'mondupe'
100
+ instance_ipaddress = $options[:ipaddress] || nil
101
+ instance_id = $options[:id] || nil
102
+ dump_tmp_path = $options[:tmpdir] || '/tmp'
103
+ expire_days = $options[:expire_days] || 3
104
+ instance_count = 1
105
+ chef_run_list = ENV['MONDUPE_CHEF_RUN_LIST'] || ""
106
+ chef_environment = ENV['MONDUPE_CHEF_ENVIRONMENT'] || "default"
107
+ ssh_key = ENV['MONDUPE_SSH_KEY'] || "~/.ssh/id_rsa"
108
+ chef_identity_file = ENV['MONDUPE_CHEF_IDENTITY_FILE'] || ssh_key
109
+ ssh_user = ENV['MONDUPE_SSH_USER'] || "ubuntu"
110
+ route53_domain = ENV['MONDUPE_ROUTE53_DOMAIN'] || nil
111
+ instance_fqdn = ( instance_name + "." + route53_domain ) unless instance_name.nil? || route53_domain.nil?
112
+ key_pair_name = ENV['MONDUPE_KEY_PAIR_NAME'] || nil
113
+ security_group = ENV['MONDUPE_SECURITY_GROUP'] || nil
114
+ s3_bucket_name = ENV['MONDUPE_S3_BUCKET_NAME'] || nil
115
+ dump_file_name = ENV['MONDUPE_DUMP_FILE_NAME'] || 'mongodb.dump.tgz'
116
+ instance_volume_size = 60
117
+
118
+ case ARGV[0]
119
+ when "create"
120
+ start_time = Time.now
121
+ total_seconds = Time.now - start_time
122
+ puts "Creating AWS EC2 Instance with MongoDB and restoring from latest production backup"
123
+ instance = Mondupe.new.create_instance(instance_name, instance_image_id, instance_type, instance_count, security_group, key_pair_name, expire_days, instance_owner, instance_volume_size)
124
+ Mondupe.new.create_dns(instance_fqdn, route53_domain, instance)
125
+ Mondupe.new.bootstrap(instance_name, instance_fqdn, instance.ip_address, chef_environment, chef_identity_file, chef_run_list, ssh_user)
126
+ Mondupe.new.get_db_dump_from_s3(instance.ip_address, s3_bucket_name, dump_tmp_path, ssh_user, dump_file_name)
127
+ Mondupe.new.restore_db(instance.ip_address, dump_tmp_path, ssh_key, ssh_user, dump_file_name)
128
+ puts " - - - Total Run Time: #{((total_seconds % 3600) / 60).to_i}m #{((total_seconds % 3600) % 60).to_i}s - - -"
129
+ when "delete"
130
+ puts "delete mongo host"
131
+ when "bootstrap"
132
+ puts "bootstrapping node"
133
+ # Find the instance and create the instance object here
134
+ Mondupe.new.bootstrap(instance_name, instance_fqdn, instance_ipaddress, chef_environment, chef_identity_file, chef_run_list, ssh_user)
135
+ when "dumps3"
136
+ puts "getting dump"
137
+ Mondupe.new.get_db_dump_from_s3(instance_ipaddress, s3_bucket_name, dump_tmp_path, ssh_user, dump_file_name)
138
+ when "restore"
139
+ puts "Restoring mongo database from dump"
140
+ Mondupe.new.restore_db(instance_ipaddress, dump_tmp_path, ssh_key, ssh_user, dump_file_name)
141
+ when "terminate"
142
+ puts "Marking instance for termination"
143
+ Mondupe.new.terminate_instance(instance_id)
144
+ when "list"
145
+ puts "Listing all instances created by MonDupe"
146
+ Mondupe.new.list_instances
147
+ when "expire"
148
+ puts "Modifying expiration days for instance"
149
+ Mondupe.new.instance_expire(instance_id, instance_name, expire_days)
150
+ else
151
+ puts opt_parser
152
+ end
data/lib/mondupe.rb ADDED
@@ -0,0 +1,142 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'aws-sdk'
4
+
5
+ class Mondupe
6
+ def create_instance(instance_name, instance_image_id, instance_type, instance_count, security_group, key_pair_name, expire_days, instance_owner, instance_volume_size)
7
+ AWS.config(:access_key_id => ENV['AWS_ACCESS_KEY_ID'], :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'], region: 'us-east-1')
8
+
9
+ ec2 = AWS::EC2.new(:access_key_id => ENV['AWS_ACCESS_KEY_ID'], :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'])
10
+
11
+ puts "Creating with - Security Group: #{security_group} Instance Owner: #{instance_owner}"
12
+
13
+ key_pair = ec2.key_pairs[key_pair_name]
14
+
15
+ # Use this to create a new security group - Can have preset options
16
+ #security_group = ec2.security_groups.create("sg_#{instance_name}")
17
+ #security_group = 'sg_my_awesome_new_instance'
18
+
19
+ instance = ec2.instances.create(
20
+ :image_id => instance_image_id,
21
+ :block_device_mappings => [{
22
+ :device_name => "/dev/sda1",
23
+ :ebs => {
24
+ :volume_size => instance_volume_size,
25
+ :delete_on_termination => true
26
+ }
27
+ }],
28
+ :instance_type => instance_type,
29
+ :count => instance_count,
30
+ :security_groups => security_group,
31
+ :key_pair => key_pair
32
+ )
33
+
34
+ created_date_time = Time.now.to_i
35
+
36
+ # Display some information about the new instance
37
+ puts "Instance '#{instance_name}' created with ID '#{instance.id}'"
38
+
39
+ instance.tag('Name', :value => instance_name)
40
+ instance.tag('owner', :value => instance_owner)
41
+ instance.tag('expire_days', :value => expire_days)
42
+ instance.tag('created', :value => created_date_time)
43
+ instance.tag('mondupe')
44
+
45
+ puts "Added tags... "
46
+ puts " Name: #{instance_name}"
47
+ puts " owner: #{instance_owner}"
48
+ puts " expires: #{expire_days}"
49
+ puts " created: #{created_date_time}"
50
+
51
+ # Wait for instance to be ready
52
+ current_state = ""
53
+ until instance.status == :running
54
+ if current_state != instance.status
55
+ puts "Status: #{instance.status.to_s}"
56
+ print "Instance coming up "
57
+ current_state = instance.status
58
+ else
59
+ print "."
60
+ sleep 1
61
+ end
62
+ end
63
+
64
+ puts ""
65
+ puts "Instance #{instance.id} is now running"
66
+ puts "Name: #{instance.tags['Name']}"
67
+ puts "IP: #{instance.ip_address}"
68
+ puts "Public DNS: #{instance.dns_name}"
69
+ instance
70
+ end
71
+
72
+ def create_dns(instance_fqdn, route53_domain, instance)
73
+ # Set up DNS through Route53
74
+ puts "Setting up Route53 DNS..."
75
+ # Check to see if record exists
76
+ route53 = AWS::Route53.new(:access_key_id => ENV['AWS_ACCESS_KEY_ID'], :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'])
77
+ zone = route53.hosted_zones.select { |z| z.name == route53_domain }.first
78
+
79
+ rrsets = AWS::Route53::HostedZone.new(zone.id).rrsets
80
+ rrset = rrsets.create(instance_fqdn, 'A', :ttl => 300, :resource_records => [{:value => instance.ip_address }])
81
+ if rrset.exists?
82
+ # Create new record if does not exist
83
+ rrset.update
84
+ else
85
+ # Update if record exists
86
+ rrset = zone.rrsets[instance_fqdn, 'A']
87
+ rrset.resource_records = [ { :value => instance.ip_address } ]
88
+ rrset.update
89
+ end
90
+ end
91
+
92
+
93
+ def bootstrap(instance_name, instance_fqdn, instance_ipaddress, chef_environment, chef_identity_file, chef_run_list, ssh_user)
94
+ # Bootstrap the new instance with chef
95
+ puts "Bootstraping node with Chef..."
96
+ puts "Running..."
97
+ puts "knife bootstrap #{instance_ipaddress} -N #{instance_fqdn[0...-1]} -E #{chef_environment} -i #{chef_identity_file} -r #{chef_run_list} -x #{ssh_user} --sudo"
98
+ sleep 30
99
+ system("knife bootstrap #{instance_ipaddress} -N #{instance_fqdn[0...-1]} -E #{chef_environment} -i #{chef_identity_file} -r #{chef_run_list} -x #{ssh_user} --sudo")
100
+ end
101
+
102
+ def get_db_dump_from_s3(instance_ip, s3_bucket_name, dump_tmp_path, ssh_user, dump_file_name)
103
+ expiration = Time.now.to_i + 400*60
104
+ s3 = AWS::S3.new(:access_key_id => ENV['AWS_ACCESS_KEY_ID'], :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'])
105
+ backups = s3.buckets[s3_bucket_name]
106
+ latest_backup = backups.objects.sort_by {|backup| backup.last_modified}.last
107
+ download_url = latest_backup.url_for(:get, :expires_in => expiration, :response_content_type => "application/json")
108
+ puts "Download URL: #{download_url}"
109
+ puts "#{Time.now.to_s} - Starting download."
110
+ puts " Please wait..."
111
+ `ssh -i ~/.ssh/DevOps.pem #{ssh_user}@#{instance_ip} "cd #{dump_tmp_path} && wget '#{download_url}' -O #{File.join(dump_tmp_path, dump_file_name)} 2&>1"`
112
+ puts "#{Time.now.to_s} - Download completed"
113
+ end
114
+
115
+ def restore_db(instance_ip, dump_tmp_path, ssh_key, ssh_user, dump_file_name)
116
+ # Restore from the database dump
117
+ # TODO - Fail the process if any step fails
118
+ puts "#{Time.now.to_s} - Dropping existing database"
119
+ `ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "echo 'db.dropDatabase()' | mongo cde_production"`
120
+ puts "#{Time.now.to_s} - Database drop complete"
121
+ puts "Restoring Mongo Database from extracted dump: #{File.join(dump_tmp_path, "cde_production")}"
122
+ `ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "cd /tmp; tar xf #{dump_file_name}; time mongorestore /tmp/cde_production"`
123
+ puts "Removing database archive file"
124
+ `ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "rm -rf /tmp/#{File.join(dump_tmp_path, dump_file_name)}"`
125
+ puts "#{Time.now.to_s} - Removing saved searches"
126
+ `ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "mongo cde_production --eval \\"db.users.update({save_searches: {$ne: null}}, {$unset: {save_searches: ''}}, {multi: true})\\""`
127
+ puts "#{Time.now.to_s} - Cleaning up our mess..."
128
+ `ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "rm -rf #{File.join(dump_tmp_path, 'cde_production')}"`
129
+ end
130
+
131
+ def terminate(instance_id)
132
+ puts "function not quite ready yet"
133
+ end
134
+
135
+ def list
136
+ puts "function not quite ready yet"
137
+ end
138
+
139
+ def expire(instance_id, instance_name, expire_days)
140
+ puts "function not quite ready yet"
141
+ end
142
+ end
metadata ADDED
@@ -0,0 +1,62 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: mondupe
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.3
5
+ platform: ruby
6
+ authors:
7
+ - Philip Hutchins
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-04-25 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: aws-sdk
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: '1.38'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: '1.38'
27
+ description: Create an AWS EC2 node and restore a MongoDB dump to it from an AWS S3
28
+ bucket
29
+ email: flipture@gmail.com
30
+ executables:
31
+ - mondupe
32
+ extensions: []
33
+ extra_rdoc_files: []
34
+ files:
35
+ - bin/mondupe
36
+ - lib/mondupe.rb
37
+ homepage: http://phutchins.com
38
+ licenses:
39
+ - MIT
40
+ metadata: {}
41
+ post_install_message:
42
+ rdoc_options: []
43
+ require_paths:
44
+ - lib
45
+ required_ruby_version: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - '>='
48
+ - !ruby/object:Gem::Version
49
+ version: '0'
50
+ required_rubygems_version: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ requirements: []
56
+ rubyforge_project:
57
+ rubygems_version: 2.2.2
58
+ signing_key:
59
+ specification_version: 4
60
+ summary: MonDupe
61
+ test_files: []
62
+ has_rdoc: