bigbang 0.0.1

data/README

@@ -0,0 +1,19 @@
+
+
+# creates the base images
+bigbang base
+
+# run the instances
+bigbang explode <universe-info.rb>
+
+# list universes
+bigbang list
+
+# kill a universe
+bigbang kill <universe-name>
+
+# resize clusters
+bigbang resize
+
+# test configuration
+bigbang test

data/bin/bigbang

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + "/../lib/bigbang"
+
+def help
+	puts "test"
+end
+
+$universe = nil
+
+def self.universe(&block)
+	$universe = u = BigBang::Universe.new
+	u.instance_eval(&block)
+end
+
+if ARGV.empty?
+	help
+	exit 1
+end
+
+case ARGV[0]
+when 'test':
+	load './universe.rb'
+	$universe.test
+when 'explode':
+	if ARGV.size < 2 then
+		puts "#{$0} explode <universe name>"
+		exit 1
+	end
+	require 'universe.rb'
+	$universe.explode(ARGV[1])
+when 'list':
+	require 'universe.rb'
+	$universe.list
+when 'kill':
+	if ARGV.size < 2 then
+		puts "#{$0} kill <universe name>"
+		exit 1
+	end
+	require 'universe.rb'
+	$universe.kill(ARGV[1])
+else
+	puts "unknown command #{ARGV[0]}"
+	exit 2
+end
+

data/lib/bigbang/cluster.rb

@@ -0,0 +1,10 @@
+require 'bigbang/instance'
+
+module BigBang
+	class Cluster < Instance
+		def initialize(name)
+			super(name)
+		end
+		attr_accessor :size
+	end
+end

data/lib/bigbang/config.rb

@@ -0,0 +1,6 @@
+module BigBang
+	class Config
+		attr_accessor :access_key_id, :secret_key, :dns_opts, :domain
+		def dns(d) @dns_opts = d; end
+	end
+end

data/lib/bigbang/dsl.rb

@@ -0,0 +1,25 @@
+require 'bigbang/config'
+require 'bigbang/instance'
+require 'bigbang/cluster'
+
+module BigBang
+	module DSL
+		def cluster(name, &block)
+			@instances << Cluster.new(name).tap do |cluster|
+				cluster.instance_eval(&block)
+			end
+		end
+		
+		def instance(name, &block)
+			@instances << Instance.new(name).tap do |i|
+				i.instance_eval(&block)
+			end
+		end
+
+		def config(&block)
+			@config = Config.new.tap do |c|
+				c.instance_eval(&block)
+			end
+		end
+	end
+end

data/lib/bigbang/ec2-git-bootstrap.rb

@@ -0,0 +1,27 @@
+class Ec2BootstrapGenerator
+	def generate_from_dir(path)
+		basedir = File.dirname(__FILE__) + "/../../"
+		tmp=%x(mktemp -d /tmp/gen-ec2-userdata.XXXXX).strip
+		%x(mkdir -p #{tmp}/data)
+		%x(cp -r #{path}/* #{tmp}/data/)
+		%x(cp #{basedir}/src/* #{tmp})
+		tmptar=%x(mktemp /tmp/gen-ec2-userdata.XXXXX.tar.gz).strip
+		%x(tar -czf #{tmptar} -C #{tmp} .)
+
+		File.new("#{basedir}/src/bootstrap.sh").read +
+			File.new(tmptar).read
+	end
+
+	def generate_from_hash(h)
+		tmp=%x(mktemp -d /tmp/gen-ec2-userdata.XXXXX).strip
+		h.each_pair do |k,v|
+			f = File.new("#{tmp}/#{k}", "w")
+			f.write(v)
+			f.close
+		end
+
+		generate_from_dir(tmp).tap do |r|
+			%x(rm -fr #{tmp})
+		end
+	end
+end

data/lib/bigbang/instance.rb

@@ -0,0 +1,13 @@
+module BigBang
+	class Instance
+		def initialize(name)
+			@name = name
+		end
+
+		attr_accessor :ami, :key_name, :type, 
+			:name, :elastic_ip, 
+			:domain, :wildcard_domain,
+			:bootstrap_repo,
+			:bootstrap_params
+	end
+end

data/lib/bigbang/provider.rb

@@ -0,0 +1,43 @@
+module BigBang
+	class Provider
+		def initialize(config)
+			@config = config
+		end
+
+		def ec2
+			return @ec2 unless @ec2.nil?
+			@ec2 = AWS::EC2::Base.new(
+				:access_key_id => @config.access_key_id, 
+				:secret_access_key => @config.secret_key)
+		end
+
+		def dns
+			return @dns unless @dns.nil?
+			@dns = Fog::DNS.new(@config.dns_opts)
+		end
+
+		def configured_zone
+			dns.zones.find { |z| z.domain == @config.domain }
+		end
+
+		def create_dns(domain, addr)
+				configured_zone.records.create(
+						:value => addr,
+						:name => domain,
+						:type => "A")
+		end
+		
+		def eips
+			aset = ec2.describe_addresses.addressesSet
+			if aset.nil?
+				return []
+			else
+				return aset.item
+			end
+		end
+
+		def free_eips
+			eips.find_all { |eip| eip.instanceId.nil? }
+		end
+	end
+end

data/lib/bigbang/universe.rb

@@ -0,0 +1,246 @@
+require 'base64'
+require 'AWS'
+require 'ap'
+require 'set'
+require 'fog'
+require 'bigbang/provider'
+require 'bigbang/instance'
+require 'bigbang/dsl'
+require 'bigbang/ec2-git-bootstrap'
+
+module BigBang
+	class Universe
+		def initialize
+			@clusters = []
+			@instances = []
+		end
+
+		include DSL
+	
+		def provider
+			return @provider unless @provider.nil?
+			@provider = Provider.new(@config)
+		end
+
+		def get_instances
+			rset = provider.ec2.describe_instances.reservationSet
+			if rset.nil?
+				return []
+			else
+				instances = []
+				rset.item.each do |reservation|
+					reservation.instancesSet.item.each do |i|
+						instances << i
+					end
+				end
+				return instances
+			end
+		end
+
+		def test
+			ap provider.eips
+			get_instances	
+			puts "ec2 access OK"
+			@instances.collect { |i| i.ami }.to_set.each do |ami|
+				begin
+					provider.ec2.describe_images(:image_id => [ami])
+				rescue AWS::InvalidAMIIDNotFound => e
+					puts "ami #{ami} not found"
+				end
+			end
+			puts "AMI's OK"
+			if provider.configured_zone.nil?
+				puts "Configured DNS domain zone not found"
+			else
+				puts "DNS domain OK"
+			end
+		end
+
+		def wait_for_eips(expected)
+			print "waiting for eips to be allocated"
+			STDOUT.flush
+			addrs = provider.free_eips
+			while(addrs.size < expected)
+				sleep(1)
+				print "."
+				STDOUT.flush
+				addrs = provider.free_eips
+			end
+			puts
+			addrs
+		end
+
+		def allocate_addresses
+			instances = @instances.find_all { |i| i.elastic_ip == true }
+			if instances.empty?
+				puts "no need to allocate elastic ips"
+				return
+			end
+			free_ips = provider.free_eips
+			toalloc = instances.size
+			blacklist = free_ips
+			if free_ips.size > 0
+				n = nil
+				if free_ips.size >= instances.size
+					n = instances.size
+				else
+					n = free_ips.size
+				end
+				confirm("Use #{n} of your #{free_ips.size} free eips?") do
+					blacklist = []
+					toalloc -= n
+				end
+			end
+			puts "need to alloc: #{toalloc}"
+			1.upto(toalloc) do |i|
+				puts "allocating eip address #{i}"
+				provider.ec2.allocate_address
+			end
+
+			addrs = wait_for_eips(free_ips.size + toalloc)
+			avail_ips = addrs.collect { |a| a.publicIp }.to_set
+			black_ips = blacklist.collect { |a| a.publicIp }.to_set
+			(avail_ips - black_ips).to_a[0,instances.size]
+		end
+
+		def running_instances
+			get_instances.find_all { |i| i.instanceState.name == "running" }
+		end
+
+		def running_instances_count(ids)
+			running_instances.count { |i| ids.include?(i.instanceId) }
+		end
+
+		def wait_for_running(instances)
+			ids = instances.collect { |i| i.instanceId }.to_set
+			print "Waiting for all instances to be in running state. "
+			STDOUT.flush
+			while running_instances_count(ids) < instances.size
+				print "."
+				sleep(5)
+				STDOUT.flush
+			end
+			puts
+		end
+
+		def run_instances(name)
+			instances = []
+			gen = Ec2BootstrapGenerator.new
+			@instances.each do |instance|
+				userdata = Base64.encode64(gen.generate_from_hash(
+					"bootstrap-repo" => instance.bootstrap_repo
+				))
+				res = provider.ec2.run_instances(
+					:image_id => instance.ami,
+					:key_name => instance.key_name,
+					:instance_type => instance.type,
+					:user_data => userdata)
+				ap res
+				res.instancesSet.item.each do |i|
+					instances << i
+					provider.ec2.create_tags(:resource_id => i.instanceId,
+						:tag => [
+								{ 'bb_name' => instance.name },
+								{ 'bb_universe' => name},
+						])
+				end
+			end
+			instances.tap do |i|
+				wait_for_running(i)
+			end
+		end
+
+		def assign_addresses(instances, addrs) 
+			if instances.size != addrs.size
+				raise "error: instances number and addresses don't match"
+			end
+	
+			instances.each_index do |i|
+				puts "associating address #{addrs[i]} " + 
+					"to instance #{instances[i].instanceId}"
+				provider.ec2.associate_address(
+					:instance_id => instances[i].instanceId,
+					:public_ip => addrs[i])
+			end
+		end
+
+		def create_dns_entries(name, instances, addrs)
+			@instances.each_index do |i|
+				domains = [@instances[i].domain]
+				if @instances[i].domain.is_a?(Array)
+					domains = @instances[i].domain
+				end
+				domains.each do |domain|
+					domain = "#{name}.#{domain}"
+					puts "creating domain #{domain}.#{@config.domain} to #{addrs[i]}"
+					provider.create_dns(domain, addrs[i])
+					if @instances[i].wildcard_domain == true
+						provider.create_dns("*.#{domain}", addrs[i])
+					end
+				end
+			end
+		end
+
+		def explode(name)
+			free_eips = allocate_addresses
+			confirm("Will assign the following ips:\n" +
+				"#{free_eips.join("\n")}\nConfirm") do
+				instances = run_instances(name)
+				assign_addresses(instances, free_eips)
+				create_dns_entries(name, instances, free_eips)
+			end
+		end
+
+		def tags
+			tag_set = provider.ec2.describe_tags(:filter => 
+					[{'key' => 'bb_universe'}]).tagSet
+			if tag_set.nil?
+				return []
+			end
+
+			tag_set.item
+		end
+
+		def list
+			universes = Set.new
+			tags.each do |tag|
+				universes << tag.value
+			end
+			universes.each do |u|
+				puts u
+			end
+		end
+
+		def confirm(msg)
+			print "#{msg}? [y/N] "
+			if STDIN.gets.strip == "y"
+				yield
+				return true
+			else
+				puts "skipping"
+				return false
+			end
+		end
+		
+		def kill_instance(tag, running)
+			instance = running.find { |i| i.instanceId == tag.resourceId }
+			if instance.nil?
+				puts "instance #{tag.resourceId} is not running. skipping"
+				return
+			end
+			confirm("kill instance #{tag.resourceId}") do
+				provider.ec2.terminate_instances(:instance_id => [tag.resourceId])
+				puts "sent termination signal to #{tag.resourceId}"
+			end
+		end
+
+		def kill(name)
+			running = running_instances
+			tags.find_all do |tag|
+				tag.value == name
+			end.each do |tag|
+				send("kill_#{tag.resourceType}", tag, running)
+			end
+		end
+	end
+end

data/lib/bigbang.rb

@@ -0,0 +1,2 @@
+
+require 'bigbang/universe'

data/src/bootstrap.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+function extract()
+{
+	SKIP=`awk '/^__TARFILE_FOLLOWS__/ { print NR + 1; exit 0; }' $0`
+
+	#remember our file name
+	THIS=`pwd`/$0
+	tmp=$(mktemp -d /tmp/bootstrap.XXXXX)
+
+	# take the tarfile and pipe it into tar
+	echo "decompressing to $tmp"
+	cd $tmp
+	tail -n +$SKIP $THIS | tar -xz
+	export EC2_GIT_BOOTSTRAP_PATH=$tmp/data
+	chmod -R a+Xr $tmp
+	./post-extract.sh
+}
+
+set -x
+log=/var/log/ec2-git-bootstrap.log
+extract > $log
+echo "EC2-GIT-BOOTSTRAP: Completed!" >> $log 
+
+#
+# place any bash script here you need.
+# Any script here will happen after the tar file extract.
+echo "Finished"
+exit 0
+
+# NOTE: Don't place any newline characters after the last line below.
+__TARFILE_FOLLOWS__

data/src/git-ssh-wrap.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec ssh -o "StrictHostKeyChecking no" -i ./data/id_rsa $1 $2

data/src/post-extract.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -x
+apt-get -y install git
+export GIT_SSH=./git-ssh-wrap.sh
+repo=$(mktemp -d /tmp/git-bootstrap.XXXXX)
+git clone $(cat data/bootstrap-repo) $repo
+cd $repo
+./run

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification 
+name: bigbang
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Giorgenes Gelatti
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-07-15 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fog
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: amazon-ec2
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+description: ""
+email: giorgenes@gmail.com
+executables: 
+- bigbang
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- src/git-ssh-wrap.sh
+- src/post-extract.sh
+- src/bootstrap.sh
+- bin/bigbang
+- README
+- lib/bigbang.rb
+- lib/bigbang/instance.rb
+- lib/bigbang/provider.rb
+- lib/bigbang/dsl.rb
+- lib/bigbang/universe.rb
+- lib/bigbang/ec2-git-bootstrap.rb
+- lib/bigbang/config.rb
+- lib/bigbang/cluster.rb
+has_rdoc: true
+homepage: http://github.com/giorgenes/bigbang/
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: ""
+test_files: []
+