mondupe 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 34fd78b237de1808b0864a03c5849ec4502a775e
+  data.tar.gz: a07ddd4d9f5bc36c9d2495158cb2661c8b460434
+SHA512:
+  metadata.gz: 3ebc59372c5391aa062007d073a8742719077a59a6084094d951fe526b01b90501a25b33e3f119fb2032710d7f108141f3988af7b86a0fe42556e25bc8ad2c4b
+  data.tar.gz: 455bb2195c14051966791fb1a15effb331ecd10fdab9c55a2a1bbb6fe82058a45d5e3880e02c9da285851ff22ccf2abc6b701a98e4e33ca75b7c6ebf43fdafe4

data/bin/mondupe

@@ -0,0 +1,152 @@
+#!/usr/bin/env ruby
+
+require 'mondupe'
+require 'optparse'
+
+$options = {}
+
+ValidHostnameRegex=/(?!-)[A-Z\d-]{1,63}(?<!-)$/
+
+opt_parser = OptionParser.new do |opt|
+  opt.banner = "Usage: mondupe COMMAND [OPTIONS]"
+  opt.separator ""
+  opt.separator "Commands           Description"
+  opt.separator "      create     - create new mongo host."
+  opt.separator "                    Requires -n [name]"
+  opt.separator "                    Optional -i [image] -e [expire_days] -o [owner] -t [type] -a [ipaddress] -s [size]"
+  opt.separator "      delete     - delete existing mongo host."
+  opt.separator "                    Requires -n [name] || -d [id]"
+  opt.separator "      bootstrap  - push chef-client and configuration to the node."
+  opt.separator "                    Requires -n [name] -a [ipaddress]"
+  opt.separator "      dumps3     - retrieve the database dump from s3."
+  opt.separator "                    Requires -n [name] -a [ipaddress]"
+  opt.separator "      restore    - Restore a mongo dump that exists on a host"
+  opt.separator "                    Requires -n [name] -a [ipaddress]"
+  opt.separator "                    Optional -t [tmpdir]"
+  opt.separator "      expire     - (coming soon) Reset the expiration days of a node"
+  opt.separator "                    Requires ( -n [name] || -d [id] || -a [ipaddress] ) -e [expire_days]"
+  opt.separator "      list       - (coming soon) List all mongo hosts."
+  opt.separator "      help       - Get help."
+  opt.separator ""
+  opt.separator "Options"
+
+  opt.on("-n","--name HOSTNAME","Name of the host that you are creating") do |name|
+    if ValidHostnameRegex.match(name)
+      $options[:name] = name
+    else
+      abort("Invalid hostname")
+    end
+  end
+
+  opt.on("-i","--image IMAGENAME","Name of the AWS Image you would like to use") do |image|
+    $options[:image] = image
+  end
+
+  opt.on("-e","--expire DAYS","Number of days the instance should stay on line. - Default: 3") do |days|
+    $options[:expire_days] = days
+  end
+
+  opt.on("-o","--owner NAME","Name of the owner of the host to be created. - Default: Your Username") do |owner|
+    $options[:owner] = owner
+  end
+
+  opt.on("-t","--type INSTANCE_TYPE","Type of AWS host to create. - Default: m3.xlarge") do |type|
+    $options[:type] = type
+  end
+
+  opt.on("-a","--ipaddress IPADDRESS","IP address of node that already exists. Not to be used with creating new nodes") do |address|
+    $options[:ipaddress] = address
+  end
+
+  opt.on("-d","--id INSTANCE_ID","AWS ID of the instance, must exist already") do |id|
+    $options[:id] = id
+  end
+
+  opt.on("-t","--tmpdir TEMP_DIR","Directory on remote host that will be used for downloading the mongo dump. - Default: /tmp") do |tmpdir|
+    $options[:tmpdir] = tmpdir
+  end
+
+  opt.on("-s","--size VOLUME_SIZE","Set the root volume size in GB - Default: 60") do |size|
+    $options[:size] = size
+  end
+
+  opt.on("-h","--help","help") do
+    puts opt_parser
+  end
+
+  opt.separator "Environment Variables - Set these in your environment. They should be self explanitory."
+  opt.separator "  Required:"
+  opt.separator "   - MONDUPE_ROUTE53_DOMAIN"
+  opt.separator "   - MONDUPE_SECURITY_GROUP"
+  opt.separator "   - MONDUPE_S3_BUCKET_NAME"
+  opt.separator "   - MONDUPE_KEY_PAIR_NAME"
+  opt.separator ""
+  opt.separator "  Optional: (have sane defaults)"
+  opt.separator "   - MONDUPE_INSTANCE_IMAGE_ID"
+  opt.separator "   - MONDUPE_CHEF_RUN_LIST"
+  opt.separator "   - MONDUPE_CHEF_IDENTITY_FILE"
+  opt.separator "   - MONDUPE_CHEF_ENVIRONMENT"
+  opt.separator "   - MONDUPE_SSH_KEY"
+  opt.separator "   - MONDUPE_SSH_USER"
+  opt.separator "   - MONDUPE_DUMP_FILE_NAME"
+end
+
+opt_parser.parse!
+
+instance_image_id = $options[:image] || ENV['MONDUPE_INSTANCE_IMAGE_ID'] || "ami-018c9568"
+instance_type = $options[:type] || "m3.xlarge"
+instance_name = $options[:name] || nil
+instance_owner = $options[:owner] || ENV['USER'] || 'mondupe'
+instance_ipaddress = $options[:ipaddress] || nil
+instance_id = $options[:id] || nil
+dump_tmp_path = $options[:tmpdir] || '/tmp'
+expire_days = $options[:expire_days] || 3
+instance_count = 1
+chef_run_list = ENV['MONDUPE_CHEF_RUN_LIST'] || ""
+chef_environment = ENV['MONDUPE_CHEF_ENVIRONMENT'] || "default"
+ssh_key = ENV['MONDUPE_SSH_KEY'] || "~/.ssh/id_rsa"
+chef_identity_file = ENV['MONDUPE_CHEF_IDENTITY_FILE'] || ssh_key
+ssh_user = ENV['MONDUPE_SSH_USER'] || "ubuntu"
+route53_domain = ENV['MONDUPE_ROUTE53_DOMAIN'] || nil
+instance_fqdn = ( instance_name + "." + route53_domain ) unless instance_name.nil? || route53_domain.nil?
+key_pair_name = ENV['MONDUPE_KEY_PAIR_NAME'] || nil
+security_group = ENV['MONDUPE_SECURITY_GROUP'] || nil
+s3_bucket_name = ENV['MONDUPE_S3_BUCKET_NAME'] || nil
+dump_file_name = ENV['MONDUPE_DUMP_FILE_NAME'] || 'mongodb.dump.tgz'
+instance_volume_size = 60
+
+case ARGV[0]
+when "create"
+  start_time = Time.now
+  total_seconds = Time.now - start_time
+  puts "Creating AWS EC2 Instance with MongoDB and restoring from latest production backup"
+  instance = Mondupe.new.create_instance(instance_name, instance_image_id, instance_type, instance_count, security_group, key_pair_name, expire_days, instance_owner, instance_volume_size)
+  Mondupe.new.create_dns(instance_fqdn, route53_domain, instance)
+  Mondupe.new.bootstrap(instance_name, instance_fqdn, instance.ip_address, chef_environment, chef_identity_file, chef_run_list, ssh_user)
+  Mondupe.new.get_db_dump_from_s3(instance.ip_address, s3_bucket_name, dump_tmp_path, ssh_user, dump_file_name)
+  Mondupe.new.restore_db(instance.ip_address, dump_tmp_path, ssh_key, ssh_user, dump_file_name)
+  puts " - - - Total Run Time: #{((total_seconds % 3600) / 60).to_i}m #{((total_seconds % 3600) % 60).to_i}s - - -"
+when "delete"
+  puts "delete mongo host"
+when "bootstrap"
+  puts "bootstrapping node"
+  # Find the instance and create the instance object here
+  Mondupe.new.bootstrap(instance_name, instance_fqdn, instance_ipaddress, chef_environment, chef_identity_file, chef_run_list, ssh_user)
+when "dumps3"
+  puts "getting dump"
+  Mondupe.new.get_db_dump_from_s3(instance_ipaddress, s3_bucket_name, dump_tmp_path, ssh_user, dump_file_name)
+when "restore"
+  puts "Restoring mongo database from dump"
+  Mondupe.new.restore_db(instance_ipaddress, dump_tmp_path, ssh_key, ssh_user, dump_file_name)
+when "terminate"
+  puts "Marking instance for termination"
+  Mondupe.new.terminate_instance(instance_id)
+when "list"
+  puts "Listing all instances created by MonDupe"
+  Mondupe.new.list_instances
+when "expire"
+  puts "Modifying expiration days for instance"
+  Mondupe.new.instance_expire(instance_id, instance_name, expire_days)
+else
+  puts opt_parser
+end

data/lib/mondupe.rb

@@ -0,0 +1,142 @@
+#!/usr/bin/env ruby
+
+require 'aws-sdk'
+
+class Mondupe
+  def create_instance(instance_name, instance_image_id, instance_type, instance_count, security_group, key_pair_name, expire_days, instance_owner, instance_volume_size)
+    AWS.config(:access_key_id => ENV['AWS_ACCESS_KEY_ID'], :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'], region: 'us-east-1')
+
+    ec2 = AWS::EC2.new(:access_key_id => ENV['AWS_ACCESS_KEY_ID'], :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'])
+
+    puts "Creating with - Security Group: #{security_group} Instance Owner: #{instance_owner}"
+
+    key_pair = ec2.key_pairs[key_pair_name]
+
+    # Use this to create a new security group - Can have preset options
+    #security_group = ec2.security_groups.create("sg_#{instance_name}")
+    #security_group = 'sg_my_awesome_new_instance'
+
+    instance = ec2.instances.create(
+      :image_id => instance_image_id,
+      :block_device_mappings => [{
+        :device_name => "/dev/sda1",
+        :ebs => {
+          :volume_size => instance_volume_size,
+          :delete_on_termination => true
+        }
+      }],
+      :instance_type => instance_type,
+      :count => instance_count,
+      :security_groups => security_group,
+      :key_pair => key_pair
+    )
+
+    created_date_time = Time.now.to_i
+
+    # Display some information about the new instance
+    puts "Instance '#{instance_name}' created with ID '#{instance.id}'"
+
+    instance.tag('Name', :value => instance_name)
+    instance.tag('owner', :value => instance_owner)
+    instance.tag('expire_days', :value => expire_days)
+    instance.tag('created', :value => created_date_time)
+    instance.tag('mondupe')
+
+    puts "Added tags... "
+    puts "  Name: #{instance_name}"
+    puts "  owner: #{instance_owner}"
+    puts "  expires: #{expire_days}"
+    puts "  created: #{created_date_time}"
+
+    # Wait for instance to be ready
+    current_state = ""
+    until instance.status == :running
+      if current_state != instance.status
+        puts "Status: #{instance.status.to_s}"
+        print "Instance coming up "
+        current_state = instance.status
+      else
+        print "."
+        sleep 1
+      end
+    end
+
+    puts ""
+    puts "Instance #{instance.id} is now running"
+    puts "Name: #{instance.tags['Name']}"
+    puts "IP: #{instance.ip_address}"
+    puts "Public DNS: #{instance.dns_name}"
+    instance
+  end
+
+  def create_dns(instance_fqdn, route53_domain, instance)
+    # Set up DNS through Route53
+    puts "Setting up Route53 DNS..."
+    # Check to see if record exists
+    route53 = AWS::Route53.new(:access_key_id => ENV['AWS_ACCESS_KEY_ID'], :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'])
+    zone = route53.hosted_zones.select { |z| z.name == route53_domain }.first
+
+    rrsets = AWS::Route53::HostedZone.new(zone.id).rrsets
+    rrset = rrsets.create(instance_fqdn, 'A', :ttl => 300, :resource_records => [{:value => instance.ip_address }])
+    if rrset.exists?
+      # Create new record if does not exist
+      rrset.update
+    else
+      # Update if record exists
+      rrset = zone.rrsets[instance_fqdn, 'A']
+      rrset.resource_records = [ { :value => instance.ip_address } ]
+      rrset.update
+    end
+  end
+
+
+  def bootstrap(instance_name, instance_fqdn, instance_ipaddress, chef_environment, chef_identity_file, chef_run_list, ssh_user)
+    # Bootstrap the new instance with chef
+    puts "Bootstraping node with Chef..."
+    puts "Running..."
+    puts "knife bootstrap #{instance_ipaddress} -N #{instance_fqdn[0...-1]} -E #{chef_environment} -i #{chef_identity_file} -r #{chef_run_list} -x #{ssh_user} --sudo"
+    sleep 30
+    system("knife bootstrap #{instance_ipaddress} -N #{instance_fqdn[0...-1]} -E #{chef_environment} -i #{chef_identity_file} -r #{chef_run_list} -x #{ssh_user} --sudo")
+  end
+
+  def get_db_dump_from_s3(instance_ip, s3_bucket_name, dump_tmp_path, ssh_user, dump_file_name)
+    expiration = Time.now.to_i + 400*60
+    s3 = AWS::S3.new(:access_key_id => ENV['AWS_ACCESS_KEY_ID'], :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'])
+    backups = s3.buckets[s3_bucket_name]
+    latest_backup = backups.objects.sort_by {|backup| backup.last_modified}.last
+    download_url = latest_backup.url_for(:get, :expires_in => expiration, :response_content_type => "application/json")
+    puts "Download URL: #{download_url}"
+    puts "#{Time.now.to_s} - Starting download."
+    puts "  Please wait..."
+    `ssh -i ~/.ssh/DevOps.pem #{ssh_user}@#{instance_ip} "cd #{dump_tmp_path} && wget '#{download_url}' -O #{File.join(dump_tmp_path, dump_file_name)} 2&>1"`
+    puts "#{Time.now.to_s} - Download completed"
+  end
+
+  def restore_db(instance_ip, dump_tmp_path, ssh_key, ssh_user, dump_file_name)
+    # Restore from the database dump
+    # TODO - Fail the process if any step fails
+    puts "#{Time.now.to_s} - Dropping existing database"
+    `ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "echo 'db.dropDatabase()' | mongo cde_production"`
+    puts "#{Time.now.to_s} - Database drop complete"
+    puts "Restoring Mongo Database from extracted dump: #{File.join(dump_tmp_path, "cde_production")}"
+    `ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "cd /tmp; tar xf #{dump_file_name}; time mongorestore /tmp/cde_production"`
+    puts "Removing database archive file"
+    `ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "rm -rf /tmp/#{File.join(dump_tmp_path, dump_file_name)}"`
+    puts "#{Time.now.to_s} - Removing saved searches"
+    `ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "mongo cde_production --eval \\"db.users.update({save_searches: {$ne: null}}, {$unset: {save_searches: ''}}, {multi: true})\\""`
+    puts "#{Time.now.to_s} - Cleaning up our mess..."
+    `ssh -i #{ssh_key} #{ssh_user}@#{instance_ip} "rm -rf #{File.join(dump_tmp_path, 'cde_production')}"`
+  end
+
+  def terminate(instance_id)
+    puts "function not quite ready yet"
+  end
+
+  def list
+    puts "function not quite ready yet"
+  end
+
+  def expire(instance_id, instance_name, expire_days)
+    puts "function not quite ready yet"
+  end
+end

metadata

@@ -0,0 +1,62 @@
+--- !ruby/object:Gem::Specification
+name: mondupe
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Philip Hutchins
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.38'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.38'
+description: Create an AWS EC2 node and restore a MongoDB dump to it from an AWS S3
+  bucket
+email: flipture@gmail.com
+executables:
+- mondupe
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/mondupe
+- lib/mondupe.rb
+homepage: http://phutchins.com
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: MonDupe
+test_files: []
+has_rdoc: