ec2launcher 1.0.0

data/startup-scripts/setup.rb

@@ -0,0 +1,154 @@
+#!/usr/bin/ruby
+
+require 'rubygems'
+
+require 'optparse'
+require 'ostruct'
+
+require 'json'
+
+SETUP_SCRIPT = "setup_instance.rb"
+SETUP_SCRIPT_URL = "https://s3.amazonaws.com/startup-scripts/#{SETUP_SCRIPT}"
+
+class InitOptions
+	def initialize
+	    @opts = OptionParser.new do |opts|
+	    	opts.banner = "Usage: #{__FILE__} [SETUP.JSON] [options]"
+	    	opts.separator ""
+
+	    	opts.on("-e", "--environment ENV", "The environment for the server.") do |env|
+	    		@options.environ = env
+	    	end
+
+	    	opts.on("-a", "--application NAME", "The name of the application class for the new server.") do |app_name|
+	    		@options.application = app_name
+	    	end
+
+	    	opts.on("-h", "--hostname NAME", "The name for the new server.") do |hostname|
+	    		@options.hostname = hostname
+	    	end
+
+			opts.separator ""
+	    	opts.separator "Additional launch options:"
+
+			opts.on("-c", "--clone HOST", "Clone the latest snapshots from a specific host.") do |clone_host|
+				@options.clone_host = clone_host
+			end
+
+	      	opts.separator ""
+	      	opts.separator "Common options:"
+
+	      	# No argument, shows at tail.  This will print an options summary.
+	      	# Try it and see!
+	      	opts.on_tail("-?", "--help", "Show this message") do
+	        	puts opts
+	        	exit
+	    	end    	
+	    end
+	end
+
+	def parse(args)
+		@options = OpenStruct.new
+
+		@options.environ = nil
+		@options.application = nil
+		@options.hostname = nil
+
+		@options.clone_host = nil
+
+		@opts.parse!(args)
+		@options
+	end
+
+	def help
+		puts @opts
+	end
+end
+
+# Runs a command and displays the output line by line
+def run_command(cmd)
+  IO.popen(cmd) do |f|
+    while ! f.eof
+      puts f.gets
+    end
+  end
+end
+
+option_parser = InitOptions.new
+options = option_parser.parse(ARGV)
+
+setup_json_filename = ARGV[0]
+
+# Read the setup JSON file
+instance_data = JSON.parse(File.read(setup_json_filename))
+
+# Pre-install gems
+unless instance_data["gems"].nil?
+	instance_data["gems"].each {|gem_name| puts `/usr/bin/gem install --no-rdoc --no-ri #{gem_name}` }
+end
+
+# Pre-install packages
+unless instance_data["packages"].nil?
+	instance_data["packages"].each {|pkg_name| puts `yum install -y #{pkg_name}` }
+end
+
+
+# Load the AWS access keys
+properties = {}
+File.open(instance_data['aws_keyfile'], 'r') do |file|
+  file.read.each_line do |line|
+    line.strip!
+    if (line[0] != ?# and line[0] != ?=)
+      i = line.index('=')
+      if (i)
+        properties[line[0..i - 1].strip] = line[i + 1..-1].strip
+      else
+        properties[line] = ''
+      end
+    end
+  end
+end
+AWS_ACCESS_KEY = properties["AWS_ACCESS_KEY"].gsub('"', '')
+AWS_SECRET_ACCESS_KEY = properties["AWS_SECRET_ACCESS_KEY"].gsub('"', '')
+  
+# Create s3curl auth file
+s3curl_auth_data = <<EOF
+%awsSecretAccessKeys = (
+    # personal account
+    startup => {
+        id => '#{AWS_ACCESS_KEY}',
+        key => '#{AWS_SECRET_ACCESS_KEY}'
+    }
+);
+EOF
+
+home_folder = `echo $HOME`.strip
+File.open("#{home_folder}/.s3curl", "w") do |f|
+  f.puts s3curl_auth_data
+end
+`chmod 600 #{home_folder}/.s3curl`
+
+# Retrieve validation.pem
+puts "Retrieving Chef validation.pem ..."
+puts `s3curl.pl --id startup #{instance_data['chef_validation_pem_url']} > /etc/chef/validation.pem`
+
+# Setting hostname
+puts "Setting hostname ... #{options.hostname}"
+`hostname #{options.hostname}`
+`sed -i 's/^HOSTNAME=.*$/HOSTNAME=#{options.hostname}/' /etc/sysconfig/network`
+
+# Set Chef node name
+File.open("/etc/chef/client.rb", 'a') { |f| f.write("node_name \"#{options.hostname}\"") }
+
+# Setup Chef client
+puts "Connecting to Chef ..."
+`rm -f /etc/chef/client.pem`
+puts `chef-client`
+
+# Retrieve secondary setup script and run it
+puts "Getting role setup script ..."
+puts `s3curl.pl --id startup #{SETUP_SCRIPT_URL} > /tmp/#{SETUP_SCRIPT} && chmod +x /tmp/#{SETUP_SCRIPT}`
+command = "/tmp/#{SETUP_SCRIPT} -a #{options.application} -e #{options.environ} -h #{options.hostname} #{setup_json_filename}"
+command += " -c #{options.clone_host}" unless options.clone_host.nil?
+command += " > /var/log/cloud-init.log"
+run_command(command)

data/startup-scripts/setup_instance.rb

@@ -0,0 +1,340 @@
+#!/usr/bin/ruby
+
+require 'rubygems'
+
+require 'optparse'
+require 'ostruct'
+
+require 'json'
+
+require 'aws-sdk'
+
+AWS_KEYS = "/etc/aws/startup_runner_keys"
+
+class InitOptions
+	def initialize
+	    @opts = OptionParser.new do |opts|
+	    	opts.banner = "Usage: #{__FILE__} [SETUP.JSON] [options]"
+	    	opts.separator ""
+
+	    	opts.on("-e", "--environment ENV", "The environment for the server.") do |env|
+	    		@options.environ = env
+	    	end
+
+	    	opts.on("-a", "--application NAME", "The name of the application class for the new server.") do |app_name|
+	    		@options.application = app_name
+	    	end
+
+	    	opts.on("-h", "--hostname NAME", "The name for the new server.") do |hostname|
+	    		@options.hostname = hostname
+	    	end
+
+			opts.separator ""
+	    	opts.separator "Additional launch options:"
+
+			opts.on("-c", "--clone HOST", "Clone the latest snapshots from a specific host.") do |clone_host|
+				@options.clone_host = clone_host
+			end
+
+	      	opts.separator ""
+	      	opts.separator "Common options:"
+
+	      	# No argument, shows at tail.  This will print an options summary.
+	      	# Try it and see!
+	      	opts.on_tail("-?", "--help", "Show this message") do
+	        	puts opts
+	        	exit
+	    	end    	
+	    end
+	end
+
+	def parse(args)
+		@options = OpenStruct.new
+
+		@options.environ = nil
+		@options.application = nil
+		@options.hostname = nil
+
+		@options.clone_host = nil
+
+		@opts.parse!(args)
+		@options
+	end
+
+	def help
+		puts @opts
+	end
+end
+
+##############################
+# Wrapper that retries failed calls to AWS
+# with an exponential back-off rate.
+def retry_aws_with_backoff(&block)
+	timeout = 1
+	result = nil
+	while timeout < 33 && result.nil?
+		begin
+	  		result = yield
+		rescue AWS::Errors::ServerError
+			puts "Error contacting Amazon. Sleeping #{timeout} seconds."
+			sleep timeout
+			timeout *= 2
+			result = nil
+		end
+	end
+	result
+end
+
+
+# Runs a command and displays the output line by line
+def run_command(cmd)
+  IO.popen(cmd) do |f|
+    while ! f.eof
+      puts f.gets
+    end
+  end
+end
+
+option_parser = InitOptions.new
+options = option_parser.parse(ARGV)
+
+setup_json_filename = ARGV[0]
+
+# Load the AWS access keys
+properties = {}
+File.open(AWS_KEYS, 'r') do |file|
+  file.read.each_line do |line|
+    line.strip!
+    if (line[0] != ?# and line[0] != ?=)
+      i = line.index('=')
+      if (i)
+        properties[line[0..i - 1].strip] = line[i + 1..-1].strip
+      else
+        properties[line] = ''
+      end
+    end
+  end
+end
+AWS_ACCESS_KEY = properties["AWS_ACCESS_KEY"].gsub('"', '')
+AWS_SECRET_ACCESS_KEY = properties["AWS_SECRET_ACCESS_KEY"].gsub('"', '')
+
+##############################
+# Find current instance data
+EC2_INSTANCE_TYPE = `wget -T 5 -q -O - http://169.254.169.254/latest/meta-data/instance-type`
+
+# Read the setup JSON file
+instance_data = JSON.parse(File.read(setup_json_filename))
+
+##############################
+# Block devices
+##############################
+
+# Creates filesystem on a device
+# XFS on 64-bit
+# ext4 on 32-bit
+def format_filesystem(system_arch, device)
+	fs_type = system_arch == "x86_64" ? "XFS" : "ext4"
+	puts "Formatting #{fs_type} filesystem on #{device} ..."
+
+	command = case system_arch
+		when "x86_64" then "/sbin/mkfs.xfs -f #{device}"
+		else "/sbin/mkfs.ext4 -F #{device}"
+	end
+	IO.popen(command) do |f|
+		while ! f.eof
+			puts f.gets
+		end
+	end
+end
+
+# Creates and formats a RAID array, given a
+# list of partitioned devices
+def initialize_raid_array(system_arch, device_list, raid_device = '/dev/md0', raid_type = 0)
+	partitions = device_list.collect {|device| "#{device}1" }
+
+	puts "Creating RAID-#{raid_type.to_s} array #{raid_device} ..."
+	command = "/sbin/mdadm --create #{raid_device} --level #{raid_type.to_s} --raid-devices #{partitions.length} #{partitions.join(' ')}"
+	puts command
+	puts `#{command}`
+
+	format_filesystem(system_arch, raid_device)
+end
+
+# Creates a mount point, mounts the device and adds it to fstab
+def mount_device(device, mount_point, owner, group, fs_type)
+	puts `echo #{device} #{mount_point} #{fs_type} noatime 0 0|tee -a /etc/fstab`
+	puts `mkdir -p #{mount_point}`
+	puts `mount #{mount_point}`
+	puts `chown #{owner}:#{owner} #{mount_point}`
+end
+
+# Partitions a list of mounted EBS volumes
+def partition_devices(device_list)
+	puts "Partioning devices ..."
+	device_list.each do |device|
+		puts "  * #{device}"
+		`echo 0|sfdisk #{device}`
+	end
+
+	puts "Sleeping 10 seconds to reload partition tables ..."
+	sleep 10
+end
+
+##############################
+# Assembles a set of existing partitions into a RAID array.
+def assemble_raid_array(partition_list, raid_device = '/dev/md0', raid_type = 0)
+  mylog.info "Assembling cloned RAID-#{raid_type.to_s} array #{raid_device} ..."
+  command = "/sbin/mdadm --assemble #{raid_device} #{partition_list.join(' ')}"
+  puts command
+  puts `#{command}`
+end
+
+# Initializes a raid array with existing EBS volumes that are already attached to the instace.
+# Partitions & formats new volumes.
+# Returns the RAID device name.
+def setup_attached_raid_array(system_arch, devices, raid_device = '/dev/md0', raid_type = 0, clone = false)
+	partitions = devices.collect {|device| "#{device}1" }
+
+	unless clone
+		partition_devices(devices)
+		initialize_raid_array(system_arch, devices, raid_device, raid_type)
+	else
+		assemble_raid_array(partitions, raid_device, raid_type)
+	end
+	`echo DEVICE #{partitions.join(' ')} |tee -a /etc/mdadm.conf`
+
+  # RAID device name can be a symlink on occasion, so we
+  # want to de-reference the symlink to keep everything clear.
+	raid_info = `/sbin/mdadm --detail --scan`.split("\n")[-1].split()
+	Pathname.new(raid_info[1]).realpath.to_s
+end
+
+def build_block_devices(count, device = "xvdj", &block)
+  device_name = device
+  0.upto(count - 1).each do |index|
+    yield device_name, index
+    device_name.next!
+  end
+end
+
+system_arch = `uname -p`.strip
+default_fs_type = system_arch == "x86_64" ? "xfs" : "ext4"
+
+#  Process ephemeral devices first
+ephemeral_drive_count = case EC2_INSTANCE_TYPE
+	when "m1.small" then 1
+	when "m1.medium" then 1
+	when "m2.xlarge" then 1
+	when "m2.2xlarge" then 1
+	when "c1.medium" then 1
+	when "m1.large" then 2
+	when "m2.4xlarge" then 2
+	when "cc1.4xlarge" then 2
+	when "cg1.4xlarge" then 2
+	when "m1.xlarge" then 4
+	when "c1.xlarge" then 4
+	when "cc2.8xlarge" then 4
+	else 0
+end
+
+# Partition the ephemeral drives
+partition_list = []
+build_block_devices(ephemeral_drive_count, "xvdf") do |device_name, index|
+	partition_list << "/dev/#{device_name}"
+end
+partition_devices(partition_list)
+
+# Format and mount the ephemeral drives
+build_block_devices(ephemeral_drive_count, "xvdf") do |device_name, index|
+	format_filesystem(system_arch, "/dev/#{device_name}1")
+
+	mount_point = case index
+		when 1 then "/mnt"
+		else "/mnt/extra#{index - 1}"
+	end
+	mount_device("/dev/#{device_name}1", mount_point, "root", "root", default_fs_type)
+end
+
+# Process EBS volumes
+unless instance_data["block_devices"].nil?
+	next_device_name = "xvdj"
+	instance_data["block_devices"].each do |block_device_json|
+		if block_device_json["raid_level"].nil?
+			# If we're not cloning an existing snapshot, then we need to partition and format the drive.
+			if options.clone_host.nil?
+				partition_devices([ "/dev/#{next_device_name}" ])
+				format_filesystem(system_arch, "/dev/#{next_device_name}1")
+			end
+			mount_device("#{next_device_name}1", block_device_json["mount_point"], block_device_json["owner"], block_device_json["group"], default_fs_type)
+			next_device_name.next!
+		else
+			raid_devices = []
+			build_block_devices(block_device_json["count"], next_device_name) do |device_name, index|
+				raid_devices << "/dev/#{device_name}"
+				next_device_name = device_name
+			end
+			raid_device_name = setup_attached_raid_array(system_arch, raid_devices, "/dev/md0", block_device_json["raid_level"].to_i, ! options.clone_host.nil?)
+			mount_device(raid_device_name, block_device_json["mount_point"], block_device_json["owner"], block_device_json["group"], default_fs_type)
+		end
+	end
+end
+
+##############################
+# CHEF SETUP
+##############################
+
+##############################
+# Create knife configuration
+knife_config = <<EOF
+log_level                :info
+log_location             STDOUT
+node_name                '#{options.hostname}'
+client_key               '/etc/chef/client.pem'
+validation_client_name   'chef-validator'
+validation_key           '/etc/chef/validation.pem'
+chef_server_url          '#{instance_data["chef_server_url"]}'
+cache_type               'BasicFile'
+cache_options( :path => '/etc/chef/checksums' )
+EOF
+home_folder = `echo $HOME`.strip
+`mkdir -p #{home_folder}/.chef && chown 700 #{home_folder}/.chef`
+File.open("#{home_folder}/.chef/knife.rb", "w") do |f|
+  f.puts knife_config
+end
+`chmod 600 #{home_folder}/.chef/knife.rb`
+
+##############################
+# Add roles
+instance_data["roles"].each do |role|
+  cmd = "knife node run_list add #{options.hostname} \"role[#{role}]\""
+  puts cmd
+  puts `#{cmd}`
+end
+
+##############################
+# Launch Chef
+IO.popen("chef-client") do |f|
+  while ! f.eof
+    puts f.gets
+  end
+end
+
+##############################
+# EMAIL NOTIFICATION
+##############################
+
+unless instance_data["email_notification"].nil?
+  # Email notification through SES
+  AWS.config({
+    :access_key_id => instance_data["email_notification"]["ses_access_key"],
+    :secret_access_key => instance_data["email_notification"]["ses_secret_key"]
+  })
+  ses = AWS::SimpleEmailService.new
+  ses.send_email(
+    :from => instance_data["email_notification"]["from"],
+    :to => instance_data["email_notification"]["to"],
+    :subject => "Server setup complete: #{hostname}",
+    :body_text => "Server setup is complete for Host: #{hostname}, Environment: #{options.environ}, Application: #{options.application}",
+    :body_html => "<div>Server setup is complete for:</div><div><strong>Host:</strong> #{hostname}</div><div><strong>Environment:</strong> #{options.environ}</div><div><strong>Application:</strong> #{options.application}</div>"
+  )
+end

metadata

@@ -0,0 +1,81 @@
+--- !ruby/object:Gem::Specification
+name: ec2launcher
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Sean Laurent
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-07-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+description: Tool to manage application configurations and launch new EC2 instances
+  based on the configurations.
+email: 
+executables:
+- ec2launcher
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/ec2launcher
+- ec2launcher.gemspec
+- lib/ec2launcher.rb
+- lib/ec2launcher/application.rb
+- lib/ec2launcher/block_device.rb
+- lib/ec2launcher/block_device_builder.rb
+- lib/ec2launcher/config.rb
+- lib/ec2launcher/defaults.rb
+- lib/ec2launcher/email_notification.rb
+- lib/ec2launcher/environment.rb
+- lib/ec2launcher/init_options.rb
+- lib/ec2launcher/version.rb
+- startup-scripts/setup.rb
+- startup-scripts/setup_instance.rb
+homepage: https://github.com/StudyBlue/ec2launcher
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Tool to launch EC2 instances.
+test_files: []