adamwiggins-sumo 0.2.0

data/README.rdoc

@@ -0,0 +1,122 @@
+= Tired of wrestling with server provisioning?  Sumo!
+
+Want to fire up a one-off EC2 instance, pronto?  ec2-run-instances got you down?  Try Sumo.
+
+    $ sumo launch
+    ---> Launching instance...    i-4f809c26 (1.5s)
+    ---> Acquiring hostname...    ec2-67-202-17-178.compute-1.amazonaws.com (26.7s)
+
+    Logging you in via ssh.  Type 'exit' or Ctrl-D to return to your local system.
+    ------------------------------------------------------------------------------
+    Linux domU-12-31-39-04-31-37 2.6.21.7-2.fc8xen #1 SMP Fri Feb 15 12:39:36 EST 2008 i686
+    ...
+    root@domU-12-31-39-04-31-37:~# 
+
+Later...
+
+    $ sumo terminate
+    ec2-67-202-17-178.compute-1.amazonaws.com scheduled for termination
+
+You can manage multiple instances via "sumo list" and specifying hostname or instance id as arguments to the ssh or terminate commands.
+
+== Service installation with Chef
+
+The launch command takes an argument, which is a server role (from roles/#{role}.json inside your cookbooks repo):
+
+    $ sumo launch redis
+    ---> Launch instance...       i-b96c73d0 (1.3s)
+    ---> Acquire hostname...      ec2-75-101-191-220.compute-1.amazonaws.com (36.1s)
+    ---> Wait for ssh...          done (9.0s)
+    ---> Bootstrap chef...        done (61.3s)
+    ---> Setup redis...           done (11.9s)
+    ---> Opening firewall...      ports 6379 (5.2s)
+
+    Your instance is exporting the following resources:
+      Redis: redis://:8452cdd98f428c972f08@ec2-75-101-191-220.compute-1.amazonaws.com:6379/0
+
+The instance can assume multiple roles if you like:
+
+    $ sumo launch redis,solr,couchdb
+
+== Setup
+
+Dependencies:
+
+    $ sudo gem install amazon-ec2 thor
+
+Then create ~/.sumo/config.yml containing:
+
+    ---
+    access_id: <your amazon access key id>
+    access_secret: <your amazon secret access key>
+
+Optional config you can include any of the following in your config.yml:
+
+    user: root
+    ami: ami-ed46a784
+     availability_zone: us-east-1b
+    cookbooks_url: git://github.com/adamwiggins/chef-cookbooks.git
+
+You'll need Bacon and Mocha if you want to run the specs, and Jewler if you want to create gems.
+
+== Managing volumes
+
+Create and attach a volume to your running instance:
+
+   $ sumo create_volume
+   ---> Create 5MB volume...     vol-8a9c6ae3 (1.1s)
+   $ sumo volumes
+   vol-8a9c6ae3    5MB  available
+   $ sumo attach
+   ---> Attach vol-8a9c6ae3 to i-bc32cbd4 as /dev/sdc1... done (0.6s)
+
+Log in to format and mount the volume:
+
+   $ sumo ssh
+   root@ip-10-251-122-175:~# mkfs.ext3 /dev/sdc1
+   mke2fs 1.41.4 (27-Jan-2009)
+   Filesystem label=
+   OS type: Linux
+   Block size=4096 (log=2)
+   ...
+   $ mkdir /myvol
+   $ mount /dev/sdc1 /myvol
+   $ echo "I'm going to persist to a volume" > /myvol/hello.txt
+
+To detach from a running instance (perhaps so you can attach elsewhere):
+
+   $ sumo detatch
+   ---> Detach vol-8a9c6ae3...   done (0.6s)
+
+Destroy it if you no longer want the data stored on it:
+
+   $ sumo destroy_volume
+   ---> Destroy volume...        done (0.8s)
+
+== Some details you might want to know
+
+Sumo creates its own keypair named sumo, which is stored in ~/.ssh/keypair.pem.  Amazon doesn't let you upload your own ssh public key, which is lame, so this is the best option for making the launch-and-connect process a single step.
+
+It will also create an Amazon security group called sumo, so that it can lower the firewall for services you configure via cookbook roles.
+
+If you run any production machines from your EC2 account, I recommend setting up a separate account for use with Sumo.  It does not prompt for confirmation when terminating an instance or differentiate between instances started by it vs. instances started by other tools.
+
+== Anti-features
+
+Sumo is not a cloud management tool, a monitor tool, or anything more than a way to get an instance up right quick.  If you're looking for a way to manage a cluster of production instances, try one of these fine tools.
+
+* Pool Party
+* RightScale
+* Engine Yard Cloud
+* Cloudkick
+
+== Meta
+
+Created by Adam Wiggins
+
+Patches contributed by Orion Henry, Blake Mizerany, and Jesse Newland
+
+Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+http://github.com/adamwiggins/sumo
+

data/Rakefile

@@ -0,0 +1,23 @@
+require 'jeweler'
+
+Jeweler::Tasks.new do |s|
+	s.name = "sumo"
+	s.description = "A no-hassle way to launch one-off EC2 instances from the command line"
+	s.summary = s.description
+	s.author = "Adam Wiggins"
+	s.email = "adam@heroku.com"
+	s.homepage = "http://github.com/adamwiggins/sumo"
+	s.rubyforge_project = "sumo"
+	s.files = FileList["[A-Z]*", "{bin,lib,spec}/**/*"]
+	s.executables = %w(sumo)
+end
+
+Jeweler::RubyforgeTasks.new
+
+desc 'Run specs'
+task :spec do
+	sh 'bacon -s spec/*_spec.rb'
+end
+
+task :default => :spec
+

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/bin/sumo

@@ -0,0 +1,173 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../lib/sumo'
+
+require 'thor'
+
+class CLI < Thor
+	desc "launch [<role>]", "launch an instance as role, or omit to ssh to vanilla instance"
+	def launch(role=nil)
+		id = task("Launch instance") { sumo.launch }
+		host = task("Acquire hostname") { sumo.wait_for_hostname(id) }
+		task("Wait for ssh") { sumo.wait_for_ssh(host) }
+
+		if role
+			task("Bootstrap chef") { sumo.bootstrap_chef(host) }
+			role.split(',').each do |role|
+				task("Setup #{role}") { sumo.setup_role(host, role) }
+			end
+
+			resources = sumo.resources(host)
+			unless resources.empty?
+				task("Open firewall") do
+					ports = resources.map { |r| r.match(/:(\d+)\//)[1] }
+					ports.each { |port| sumo.open_firewall(port) }
+					"ports " + ports.join(", ")
+				end
+			end
+
+			puts
+			display_resources(host)
+		else
+			puts "\nLogging you in via ssh.  Type 'exit' or Ctrl-D to return to your local system."
+			puts '-' * 78
+			connect_ssh(host)
+		end
+	end
+
+	desc "ssh [<instance_id or hostname>]", "ssh to a specified instance or first available"
+	def ssh(id=nil)
+		inst = sumo.find(id) || sumo.running.first || abort("No running instances")
+		hostname = inst[:hostname] || wait_for_hostname(inst[:instance_id])
+		connect_ssh hostname
+	end
+
+	desc "resources [<instance_id or hostname>]", "show resources exported by an instance"
+	def resources(id=nil)
+		inst = sumo.find(id) || sumo.running.first || abort("No running instances")
+		hostname = inst[:hostname] || wait_for_hostname(inst[:instance_id])
+		display_resources(inst[:hostname])
+	end
+
+	desc "bootstrap", "bootstrap chef and cookbooks"
+	def bootstrap(id=nil)
+		inst = sumo.find(id) || sumo.running.first || abort("No running instances")
+		task "Bootstrap chef" do
+			sumo.bootstrap_chef(inst[:hostname])
+		end
+	end
+
+	desc "role", "setup instance as a role"
+	def role(role, id=nil)
+		inst = sumo.find(id) || sumo.running.first || abort("No running instances")
+		task "Setup #{role}" do
+			sumo.setup_role(inst[:hostname], role)
+		end
+	end
+
+	desc "list", "list running instances"
+	def list
+		sumo.list.each do |inst|
+			printf "%-50s %-12s %s\n", inst[:hostname], inst[:instance_id], inst[:status]
+		end
+	end
+
+	desc "console [<instance_id or hostname>]", "get console output for instance or first available"
+	def console(id=nil)
+		inst = sumo.find(id) || (sumo.running | sumo.pending).first || abort("No running or pending instances")
+
+		puts sumo.console_output(inst[:instance_id]).inspect
+	end
+
+	desc "terminate [<instance_id or hostname>]", "terminate specified instance or first available"
+	def terminate(id=nil)
+		inst = sumo.find(id) || (sumo.running | sumo.pending).first || abort("No running or pending instances")
+
+		sumo.terminate(inst[:instance_id])
+		puts "#{inst[:hostname] || inst[:instance_id]} scheduled for termination"
+	end
+
+	desc "terminate_all", "terminate all instances"
+	def terminate_all
+		instances = (sumo.running | sumo.pending)
+		abort("No running or pending instances") if instances.empty?
+		instances.each do |inst|
+			sumo.terminate(inst[:instance_id])
+			puts "#{inst[:hostname] || inst[:instance_id]} scheduled for termination"
+		end
+	end
+
+	desc "volumes", "list all volumes"
+	def volumes
+		sumo.volumes.each do |v|
+			printf "%-10s %4sMB %10s %15s %15s\n", v[:volume_id], v[:size], v[:status], v[:instance], v[:device]
+		end
+	end
+
+	desc "create_volume [<megabytes>]", "create a volume"
+	def create_volume(size=5)
+		task("Create #{size}MB volume") { sumo.create_volume(size) }
+	end
+
+	desc "destroy_volume [<volume_id>]", "destroy a volume"
+	def destroy_volume(volume=nil)
+		vol_id = (sumo.find_volume(volume) || sumo.nondestroyed_volumes.first || abort("No volumes"))[:volume_id]
+		task("Destroy volume") { sumo.destroy_volume(vol_id) }
+	end
+
+	desc "attach [<volume_id>] [<instance_id or hostname>] [<device>]", "attach volume to running instance"
+	def attach(volume=nil, inst_id=nil, device=nil)
+		vol_id = (sumo.find_volume(volume) || sumo.available_volumes.first || abort("No available volumes"))[:volume_id]
+		inst_id = (sumo.find(inst_id) || sumo.running.first || abort("No running instances"))[:instance_id]
+		device ||= '/dev/sdc1'
+		task("Attach #{vol_id} to #{inst_id} as #{device}") do
+			sumo.attach(vol_id, inst_id, device)
+		end
+	end
+
+	desc "detach [<volume_id>]", "detach volume from instance"
+	def detach(volume=nil)
+		vol_id = (sumo.find_volume(volume) || sumo.attached_volumes.first || abort("No attached volumes"))[:volume_id]
+		task("Detach #{vol_id}") { sumo.detach(vol_id) }
+	end
+
+	no_tasks do
+		def sumo
+			@sumo ||= Sumo.new
+		end
+
+		def config
+			sumo.config
+		end
+
+		def task(msg, &block)
+			printf "---> %-24s ", "#{msg}..."
+			start = Time.now
+			result = block.call || 'done'
+			finish = Time.now
+			time = sprintf("%0.1f", finish - start)
+			puts "#{result} (#{time}s)"
+			result
+		end
+
+		def connect_ssh(hostname)
+			sumo.wait_for_ssh(hostname)
+			system "ssh -i #{sumo.keypair_file} #{config['user']}@#{hostname}"
+			if $?.success?
+				puts "\nType 'sumo terminate' if you're done with this instance."
+			end
+		end
+
+		def display_resources(host)
+			resources = sumo.resources(host)
+			unless resources.empty?
+				puts "Your instance is exporting the following resources:"
+				resources.each do |resource|
+					puts "  #{resource}"
+				end
+			end
+		end
+	end
+end
+
+CLI.start

data/lib/sumo.rb

@@ -0,0 +1,268 @@
+require 'AWS'
+require 'yaml'
+require 'socket'
+
+class Sumo
+	def launch
+		ami = config['ami']
+		raise "No AMI selected" unless ami
+
+		create_keypair unless File.exists? keypair_file
+
+		create_security_group
+		open_firewall(22)
+
+		result = ec2.run_instances(
+			:image_id => ami,
+			:instance_type => config['instance_size'] || 'm1.small',
+			:key_name => 'sumo',
+			:group_id => [ 'sumo' ],
+			:availability_zone => config['availability_zone'],
+		)
+		result.instancesSet.item[0].instanceId
+	end
+
+	def list
+		@list ||= fetch_list
+	end
+
+	def volumes
+		result = ec2.describe_volumes
+		return [] unless result.volumeSet
+
+		result.volumeSet.item.map do |row|
+			{
+				:volume_id => row["volumeId"],
+				:size => row["size"],
+				:status => row["status"],
+				:device => (row["attachmentSet"]["item"].first["device"] rescue ""),
+				:instance_id => (row["attachmentSet"]["item"].first["instanceId"] rescue ""),
+			}
+		end
+	end
+
+	def available_volumes
+		volumes.select { |vol| vol[:status] == 'available' }
+	end
+
+	def attached_volumes
+		volumes.select { |vol| vol[:status] == 'in-use' }
+	end
+
+	def nondestroyed_volumes
+		volumes.select { |vol| vol[:status] != 'deleting' }
+	end
+
+	def attach(volume, instance, device)
+		result = ec2.attach_volume(
+			:volume_id => volume,
+			:instance_id => instance,
+			:device => device
+		)
+		"done"
+	end
+
+	def detach(volume)
+		result = ec2.detach_volume(:volume_id => volume, :force => "true")
+		"done"
+	end
+
+	def create_volume(size)
+		result = ec2.create_volume(
+			:availability_zone => config['availability_zone'],
+			:size => size.to_s
+		)
+		result["volumeId"]
+	end
+
+	def destroy_volume(volume)
+		ec2.delete_volume(:volume_id => volume)
+		"done"
+	end
+
+	def fetch_list
+		result = ec2.describe_instances
+		return [] unless result.reservationSet
+
+		instances = []
+		result.reservationSet.item.each do |r|
+			r.instancesSet.item.each do |item|
+				instances << {
+					:instance_id => item.instanceId,
+					:status => item.instanceState.name,
+					:hostname => item.dnsName
+				}
+			end
+		end
+		instances
+	end
+
+	def find(id_or_hostname)
+		return unless id_or_hostname
+		id_or_hostname = id_or_hostname.strip.downcase
+		list.detect do |inst|
+			inst[:hostname] == id_or_hostname or
+			inst[:instance_id] == id_or_hostname or
+			inst[:instance_id].gsub(/^i-/, '') == id_or_hostname
+		end
+	end
+
+	def find_volume(volume_id)
+		return unless volume_id
+		volume_id = volume_id.strip.downcase
+		volumes.detect do |volume|
+			inst[:volume_id] == volume_id or
+			inst[:volume_id].gsub(/^vol-/, '') == volume_id
+		end
+	end
+
+	def running
+		list_by_status('running')
+	end
+
+	def pending
+		list_by_status('pending')
+	end
+
+	def list_by_status(status)
+		list.select { |i| i[:status] == status }
+	end
+
+	def instance_info(instance_id)
+		fetch_list.detect do |inst|
+			inst[:instance_id] == instance_id
+		end
+	end
+
+	def wait_for_hostname(instance_id)
+		raise ArgumentError unless instance_id and instance_id.match(/^i-/)
+		loop do
+			if inst = instance_info(instance_id)
+				if hostname = inst[:hostname]
+					return hostname
+				end
+			end
+			sleep 1
+		end
+	end
+
+	def wait_for_ssh(hostname)
+		raise ArgumentError unless hostname
+		loop do
+			begin
+				Timeout::timeout(4) do
+					TCPSocket.new(hostname, 22)
+					return
+				end
+			rescue SocketError, Timeout::Error, Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+			end
+		end
+	end
+
+	def bootstrap_chef(hostname)
+		commands = [
+			'apt-get update',
+			'apt-get autoremove -y',
+			'apt-get install -y ruby ruby-dev rubygems git-core',
+			'gem sources -a http://gems.opscode.com',
+			'gem install chef ohai --no-rdoc --no-ri',
+			"git clone #{config['cookbooks_url']}",
+		]
+		ssh(hostname, commands)
+	end
+
+	def setup_role(hostname, role)
+		commands = [
+			"cd chef-cookbooks",
+			"/var/lib/gems/1.8/bin/chef-solo -c config.json -j roles/#{role}.json"
+		]
+		ssh(hostname, commands)
+	end
+
+	def ssh(hostname, cmds)
+		IO.popen("ssh -i #{keypair_file} #{config['user']}@#{hostname} > ~/.sumo/ssh.log 2>&1", "w") do |pipe|
+			pipe.puts cmds.join(' && ')
+		end
+		unless $?.success?
+			abort "failed\nCheck ~/.sumo/ssh.log for the output"
+		end
+	end
+
+	def resources(hostname)
+		@resources ||= {}
+		@resources[hostname] ||= fetch_resources(hostname)
+	end
+
+	def fetch_resources(hostname)
+		cmd = "ssh -i #{keypair_file} #{config['user']}@#{hostname} 'cat /root/resources' 2>&1"
+		out = IO.popen(cmd, 'r') { |pipe| pipe.read }
+		abort "failed to read resources, output:\n#{out}" unless $?.success?
+		parse_resources(out, hostname)
+	end
+
+	def parse_resources(raw, hostname)
+		raw.split("\n").map do |line|
+			line.gsub(/localhost/, hostname)
+		end
+	end
+
+	def terminate(instance_id)
+		ec2.terminate_instances(:instance_id => [ instance_id ])
+	end
+
+	def console_output(instance_id)
+		ec2.get_console_output(:instance_id => instance_id)["output"]
+	end
+
+	def config
+		@config ||= default_config.merge read_config
+	end
+
+	def default_config
+		{
+			'user' => 'root',
+			'ami' => 'ami-ed46a784',
+			'availability_zone' => 'us-east-1b',
+		}
+	end
+
+	def sumo_dir
+		"#{ENV['HOME']}/.sumo"
+	end
+
+	def read_config
+		YAML.load File.read("#{sumo_dir}/config.yml")
+	rescue Errno::ENOENT
+		raise "Sumo is not configured, please fill in ~/.sumo/config.yml"
+	end
+
+	def keypair_file
+		"#{sumo_dir}/keypair.pem"
+	end
+
+	def create_keypair
+		keypair = ec2.create_keypair(:key_name => "sumo").keyMaterial
+		File.open(keypair_file, 'w') { |f| f.write keypair }
+		File.chmod 0600, keypair_file
+	end
+
+	def create_security_group
+		ec2.create_security_group(:group_name => 'sumo', :group_description => 'Sumo')
+	rescue AWS::InvalidGroupDuplicate
+	end
+
+	def open_firewall(port)
+		ec2.authorize_security_group_ingress(
+			:group_name => 'sumo',
+			:ip_protocol => 'tcp',
+			:from_port => port,
+			:to_port => port,
+			:cidr_ip => '0.0.0.0/0'
+		)
+	rescue AWS::InvalidPermissionDuplicate
+	end
+
+	def ec2
+		@ec2 ||= AWS::EC2::Base.new(:access_key_id => config['access_id'], :secret_access_key => config['access_secret'])
+	end
+end

data/spec/base.rb

@@ -0,0 +1,21 @@
+require File.dirname(__FILE__) + '/../lib/sumo'
+
+require 'bacon'
+require 'mocha/standalone'
+require 'mocha/object'
+
+class Bacon::Context
+	include Mocha::API
+
+	def initialize(name, &block)
+		@name = name
+		@before, @after = [
+			[lambda { mocha_setup }],
+			[lambda { mocha_verify ; mocha_teardown }]
+		]
+		@block = block
+	end
+
+	def xit(desc, &bk)
+	end
+end

data/spec/sumo_spec.rb

@@ -0,0 +1,31 @@
+require File.dirname(__FILE__) + '/base'
+
+require 'fileutils'
+
+describe Sumo do
+	before do
+		@work_path = "/tmp/spec_#{Process.pid}/"
+		FileUtils.mkdir_p(@work_path)
+		File.open("#{@work_path}/config.yml", "w") do |f|
+			f.write YAML.dump({})
+		end
+
+		@sumo = Sumo.new
+		@sumo.stubs(:sumo_dir).returns(@work_path)
+	end
+
+	after do
+		FileUtils.rm_rf(@work_path)
+	end
+
+	it "defaults to user root if none is specified in the config" do
+		@sumo.config['user'].should == 'root'
+	end
+
+	it "uses specified user if one is in the config" do
+		File.open("#{@work_path}/config.yml", "w") do |f|
+			f.write YAML.dump('user' => 'joe')
+		end
+		@sumo.config['user'].should == 'joe'
+	end
+end

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification 
+name: adamwiggins-sumo
+version: !ruby/object:Gem::Version 
+  version: 0.2.0
+platform: ruby
+authors: 
+- Adam Wiggins
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-09-07 00:00:00 -07:00
+default_executable: sumo
+dependencies: []
+
+description: A no-hassle way to launch one-off EC2 instances from the command line
+email: adam@heroku.com
+executables: 
+- sumo
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- README.rdoc
+- Rakefile
+- VERSION
+- bin/sumo
+- lib/sumo.rb
+- spec/base.rb
+- spec/sumo_spec.rb
+has_rdoc: true
+homepage: http://github.com/adamwiggins/sumo
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: sumo
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: A no-hassle way to launch one-off EC2 instances from the command line
+test_files: 
+- spec/base.rb
+- spec/sumo_spec.rb