chef-vpc-toolkit 2.0.0

data/lib/chef-vpc-toolkit/version.rb

@@ -0,0 +1,8 @@
+module ChefVPCToolkit
+
+class Version
+  CHEF_VPC_TOOLKIT_ROOT = File.dirname(File.expand_path("../", File.dirname(__FILE__)))
+  VERSION = IO.read(File.join(CHEF_VPC_TOOLKIT_ROOT, 'VERSION'))
+end
+
+end

data/rake/chef_vpc_toolkit.rake

@@ -0,0 +1,284 @@
+#require 'chef-cloud-toolkit'
+
+namespace :group do
+	TMP_SG=File.join(CHEF_VPC_PROJECT, 'tmp', 'server_groups')
+
+	directory TMP_SG
+
+	desc "Create a new group of cloud servers"
+	task :create => [ TMP_SG, "chef:validate_json" ] do
+
+		request=CloudServersVPC.server_group_xml
+		configs=Util.load_configs
+
+		resp=HttpUtil.post(
+			configs["cloud_servers_vpc_url"]+"/server_groups.xml",
+			request,
+			configs["cloud_servers_vpc_username"],
+			configs["cloud_servers_vpc_password"]
+		)
+
+		hash=CloudServersVPC.server_group_hash(resp)	
+		out_file=hash["id"]+".xml"
+		File.open(File.join(TMP_SG, out_file), 'w') { |f| f.write(resp) }
+		puts "Cloud server group ID #{hash['id']} created."
+		
+	end
+
+	desc "List existing cloud server groups"
+	task :list => TMP_SG do
+
+		server_groups=[]
+		Dir[File.join(TMP_SG, '*.xml')].each do  |file|
+			server_groups << CloudServersVPC.server_group_hash(IO.read(file))
+		end
+		if server_groups.size > 0
+			puts "Cloud server groups:"
+			server_groups.sort { |a,b| b["id"] <=> a["id"] }.each do |sg|
+				gw=sg['vpn-gateway'].nil? ? "" : " (#{sg['vpn-gateway']})"
+				puts "\t#{sg['id']}: #{sg['name']}#{gw}"
+			end
+		else
+			puts "No server groups."
+		end
+
+	end
+
+	desc "Print information for a cloud server group"
+	task :show => TMP_SG do
+		id=ENV['GROUP_ID']
+		configs=Util.load_configs
+		xml=CloudServersVPC.server_group_xml_for_id(configs, File.join(TMP_SG, '*.xml'), id)
+
+		hash=CloudServersVPC.server_group_hash(xml)
+		File.open(File.join(TMP_SG, "#{hash['id']}.xml"), 'w') { |f| f.write(xml) }
+		CloudServersVPC.print_server_group(hash)
+
+	end
+
+	desc "Delete a cloud server group"
+	task :delete do
+		id=ENV['GROUP_ID']
+		configs=Util.load_configs
+		hash=Util.hash_for_group
+		if id.nil? then
+			id=hash["id"]
+		end
+		SshUtil.remove_known_hosts_ip(hash["vpn-gateway"])	
+		puts "Deleting cloud server group ID: #{id}."
+		HttpUtil.delete(
+			configs["cloud_servers_vpc_url"]+"/server_groups/#{id}.xml",
+			configs["cloud_servers_vpc_username"],
+			configs["cloud_servers_vpc_password"]
+		)
+		File.delete(File.join(TMP_SG, "#{id}.xml"))
+	end
+
+	desc "Force clean the cached server group files"
+	task :force_clean do
+		puts "Removing cached server group files."
+		FileUtils.rm_rf(TMP_SG)
+	end
+
+	desc "Poll/loop until a server group is online"
+	task :poll do
+		timeout=ENV['TIMEOUT']
+		if timeout.nil? or timeout.empty? then
+			timeout=1500 # defaults to 24 minutes
+		end
+		hash=Util.hash_for_group
+		puts "Polling for server(s) to come online (this may take a couple minutes)..."
+		servers=nil
+		vpn_gateway=nil
+		CloudServersVPC.poll_until_online(hash["id"], timeout) do |server_group_hash|
+			if servers != server_group_hash then
+				servers = server_group_hash
+				vpn_gateway = server_group_hash["vpn-gateway"] if server_group_hash["vpn-gateway"]
+				if not vpn_gateway.nil? and not vpn_gateway.empty? then
+					SshUtil.remove_known_hosts_ip(vpn_gateway)
+				end
+				ENV["GROUP_ID"]=server_group_hash['id']
+				CloudServersVPC.print_server_group(server_group_hash)
+			end
+		end
+		Rake::Task['group:show'].invoke
+		puts "Cloud server group online."
+	end
+
+end
+
+namespace :server do
+
+	desc "Rebuild a server in a server group."
+	task :rebuild => TMP_SG do
+		id=ENV['GROUP_ID']
+		server_name=ENV['SERVER_NAME']
+		raise "Please specify a SERVER_NAME." if server_name.nil?
+		configs=Util.load_configs
+
+		xml=CloudServersVPC.server_group_xml_for_id(configs, File.join(TMP_SG, '*.xml'), id)
+		hash=CloudServersVPC.server_group_hash(xml)
+		CloudServersVPC.rebuild(hash, server_name)
+
+	end
+
+end
+
+namespace :chef do
+
+	desc "Validate the Chef JSON config file."
+	task :validate_json do
+
+		configs=ChefInstaller.load_configs
+		ChefInstaller.validate_json(configs)
+
+	end
+
+	desc "Install and configure Chef on the server group"
+	task :install do
+
+		configs=ChefInstaller.load_configs
+		configs.merge!(Util.load_configs)
+		hash=Util.hash_for_group(configs)
+		os_types=CloudServersVPC.os_types(hash)
+		configs["ssh_gateway_ip"]=hash["vpn-gateway"]
+		client_validation_key=ChefInstaller.install_chef_server(configs, os_types)
+		ChefInstaller.create_databags(configs)
+		ChefInstaller.install_chef_clients(configs, client_validation_key, os_types)
+
+	end
+
+	desc "Tail the Chef client logs"
+	task :tail_logs do
+		
+		lines=ENV['LINES']
+		if lines.nil? or lines.empty? then
+			lines=100
+		end
+		configs=ChefInstaller.load_configs
+		hash=Util.hash_for_group(configs)
+		CloudServersVPC.server_names(hash) do |name|
+			puts "================================================================================"
+			puts "SERVER NAME: #{name}"
+			puts ChefInstaller.tail_log(hash["vpn-gateway"], name, "/var/log/chef/client.log", lines)
+		end
+
+	end
+
+	desc "Sync the local cookbook repos directory to the Chef server."
+	task :sync_repos do
+
+		configs=ChefInstaller.load_configs
+		hash=Util.hash_for_group(configs)
+		configs["ssh_gateway_ip"]=hash["vpn-gateway"]
+		ChefInstaller.rsync_cookbook_repos(configs)
+
+	end
+
+	desc "Create/Update databags on the Chef server."
+	task :databags do
+
+		configs=ChefInstaller.load_configs
+		hash=Util.hash_for_group(configs)
+		configs["ssh_gateway_ip"]=hash["vpn-gateway"]
+		ChefInstaller.create_databags(configs)
+
+	end
+
+end
+
+namespace :share do
+
+	desc "Sync the share data."
+	task :sync do
+
+		if File.exists?("#{CHEF_VPC_PROJECT}/share/") then
+			puts "Syncing share data."
+			configs=Util.load_configs
+			hash=Util.hash_for_group(configs)
+			system("rsync -azL '#{CHEF_VPC_PROJECT}/share/' root@#{hash['vpn-gateway']}:/mnt/share/")
+		end
+
+	end
+
+end
+
+desc "SSH into the most recently created VPN gateway server."
+task :ssh do
+	hash=Util.hash_for_group
+	exec("ssh root@#{hash['vpn-gateway']}")
+end
+
+desc "Create a server group, install chef, sync share data and cookbooks."
+task :create do
+
+	Rake::Task['group:create'].invoke
+	Rake::Task['group:poll'].invoke
+	Rake::Task['chef:sync_repos'].invoke
+	Rake::Task['chef:install'].invoke
+	#Rake::Task['share:sync'].invoke
+
+end
+
+desc "Rebuild and Re-Chef the specified server."
+task :rechef => [ "server:rebuild", "group:poll" ] do
+	server_name=ENV['SERVER_NAME']
+	raise "Please specify a SERVER_NAME." if server_name.nil?
+
+	configs=ChefInstaller.load_configs
+	configs.merge!(Util.load_configs)
+	hash=Util.hash_for_group(configs)
+	os_types=CloudServersVPC.os_types(hash)
+	configs["ssh_gateway_ip"]=hash["vpn-gateway"]
+	ChefInstaller.knife_readd_node(configs, server_name)
+	client_validation_key=ChefInstaller.client_validation_key(configs)
+	ChefInstaller.install_chef_client(configs, server_name, client_validation_key, os_types[server_name])
+
+end
+
+desc "Print help and usage information"
+task :usage do
+
+	puts ""
+	puts "Cloud Toolkit Version: #{ChefVPCToolkit::Version::VERSION}"
+	puts ""
+	puts "The following tasks are available:"
+
+	puts %x{cd #{CHEF_VPC_PROJECT} && rake -T}
+	puts "----"
+	puts "Example commands:"
+	puts ""
+	puts "\t- Create a new cloud server group, upload cookbooks, install chef\n\ton all the nodes, sync share data and cookbooks."
+	puts ""
+	puts "\t\t$ rake create"
+
+	puts ""
+	puts "\t- List your currently running cloud server groups."
+	puts ""
+	puts "\t\t$ rake group:list"
+
+	puts ""
+	puts "\t- SSH into the current (most recently created) cloud server group"
+	puts ""
+	puts "\t\t$ rake ssh"
+
+	puts ""
+	puts "\t- SSH into a cloud server group with an ID of 3"
+	puts ""
+	puts "\t\t$ rake ssh GROUP_ID=3"
+
+	puts ""
+	puts "\t- Delete the cloud server group with an ID of 3"
+	puts ""
+	puts "\t\t$ rake group:delete GROUP_ID=3"
+
+	puts ""
+	puts "\t- Rebuild/Re-Chef a server in the most recently created cloud\n\tserver group"
+	puts ""
+	puts "\t\t$ rake rechef SERVER_NAME=db1"
+
+	puts ""
+
+end
+
+task :default => 'usage'

data/test/cloud_servers_vpc_test.rb

@@ -0,0 +1,174 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+require 'tempfile'
+
+module ChefVPCToolkit
+
+class CloudServersVPCTest < Test::Unit::TestCase
+
+SERVER_GROUP_XML = %{
+<?xml version="1.0" encoding="UTF-8"?>
+<server-group>
+  <created-at type="datetime">2010-10-15T15:15:58-04:00</created-at>
+  <description>test description</description>
+  <domain-name>mydomain.net</domain-name>
+  <historical type="boolean">false</historical>
+  <id type="integer">1759</id>
+  <last-used-ip-address>172.19.0.2</last-used-ip-address>
+  <name>test</name>
+  <owner-name>dan.prince</owner-name>
+  <updated-at type="datetime">2010-10-15T15:15:58-04:00</updated-at>
+  <user-id type="integer">3</user-id>
+  <vpn-network>172.19.0.0</vpn-network>
+  <vpn-subnet>255.255.128.0</vpn-subnet>
+  <servers type="array">
+    <server>
+      <account-id type="integer">3</account-id>
+      <cloud-server-id-number type="integer">1</cloud-server-id-number>
+      <created-at type="datetime">2010-10-15T15:15:58-04:00</created-at>
+      <description>login1</description>
+      <error-message nil="true"></error-message>
+      <external-ip-addr>184.106.205.120</external-ip-addr>
+      <flavor-id type="integer">4</flavor-id>
+      <historical type="boolean">false</historical>
+      <id type="integer">5513</id>
+      <image-id type="integer">14</image-id>
+      <internal-ip-addr>10.179.107.203</internal-ip-addr>
+      <name>login1</name>
+      <openvpn-server type="boolean">true</openvpn-server>
+      <retry-count type="integer">0</retry-count>
+      <server-group-id type="integer">1759</server-group-id>
+      <status>Online</status>
+      <updated-at type="datetime">2010-10-15T15:18:22-04:00</updated-at>
+      <vpn-network-interfaces type="array"/>
+    </server>
+    <server>
+      <account-id type="integer">3</account-id>
+      <cloud-server-id-number type="integer">2</cloud-server-id-number>
+      <created-at type="datetime">2010-10-15T15:15:58-04:00</created-at>
+      <description>test1</description>
+      <error-message nil="true"></error-message>
+      <external-ip-addr>184.106.205.121</external-ip-addr>
+      <flavor-id type="integer">49</flavor-id>
+      <historical type="boolean">false</historical>
+      <id type="integer">5513</id>
+      <image-id type="integer">49</image-id>
+      <internal-ip-addr>10.179.107.204</internal-ip-addr>
+      <name>test1</name>
+      <openvpn-server type="boolean">false</openvpn-server>
+      <retry-count type="integer">0</retry-count>
+      <server-group-id type="integer">1759</server-group-id>
+      <status>Online</status>
+      <updated-at type="datetime">2010-10-15T15:18:22-04:00</updated-at>
+      <vpn-network-interfaces type="array"/>
+    </server>
+  </servers>
+</server-group>
+}
+
+  def test_os_types
+
+    #response = mock()
+    #response.stubs(:code => "200", :body => json_response)
+
+    #@conn.stubs(:csreq).returns(response)
+    hash=CloudServersVPC.server_group_hash(SERVER_GROUP_XML)
+    os_types=CloudServersVPC.os_types(hash)
+
+    assert_equal 2, os_types.size
+    assert_equal "rhel", os_types["login1"]
+    assert_equal "ubuntu", os_types["test1"]
+
+  end
+
+  def test_server_names
+
+    hash=CloudServersVPC.server_group_hash(SERVER_GROUP_XML)
+    names=CloudServersVPC.server_names(hash)
+
+    assert_equal 2, names.size
+    assert names.include?("login1")
+    assert names.include?("test1")
+
+  end
+
+  def test_print_server_group
+
+	hash=CloudServersVPC.server_group_hash(SERVER_GROUP_XML)
+	tmp = Tempfile.open('chef-cloud-toolkit')
+	begin
+		$stdout = tmp
+		CloudServersVPC.print_server_group(hash)
+		tmp.flush
+		output=IO.read(tmp.path)
+		$stdout = STDOUT
+		assert output =~ /login1/
+		assert output =~ /test1/
+		assert output =~ /184.106.205.120/
+	ensure
+		$stdout = STDOUT
+	end
+
+  end
+
+  def test_most_recent_server_group_hash
+
+	tmp_dir=TmpDir.new_tmp_dir
+	File.open("#{tmp_dir}/5.xml", 'w') do |f|
+		f.write(SERVER_GROUP_XML)
+	end
+
+	hash=CloudServersVPC.most_recent_server_group_hash(File.join(tmp_dir, '*.xml'))
+
+	assert_equal "mydomain.net", hash["domain-name"]
+	assert_equal "1759", hash["id"]
+	assert_equal 2, hash["servers"].size
+
+  end
+
+  def test_server_group_xml_for_id
+
+	tmp_dir=TmpDir.new_tmp_dir
+	File.open("#{tmp_dir}/5.xml", 'w') do |f|
+		f.write(SERVER_GROUP_XML)
+	end
+
+	configs={
+		"cloud_servers_vpc_url" => "http://localhost/",
+		"cloud_servers_vpc_username" => "admin",
+		"cloud_servers_vpc_password" => "test123"
+	}
+    HttpUtil.stubs(:get).returns(SERVER_GROUP_XML)
+	xml=CloudServersVPC.server_group_xml_for_id(configs, File.join(tmp_dir, '*.xml'))
+	assert_not_nil xml
+	xml=CloudServersVPC.server_group_xml_for_id(configs, File.join(tmp_dir, '*.xml'),"1759")
+	assert_not_nil xml
+
+  end
+
+  def test_load_public_key
+
+	key=CloudServersVPC.load_public_key
+	assert_not_nil key
+
+  end
+
+  def test_rebuild
+
+    response={}
+    response.stubs(:code).returns('200')
+    HttpUtil.stubs(:post).returns(response)
+
+    hash=CloudServersVPC.server_group_hash(SERVER_GROUP_XML)
+
+    assert_raises(RuntimeError) do
+		CloudServersVPC.rebuild(hash, "login1")
+	end
+
+	assert "200", CloudServersVPC.rebuild(hash, "test1").code
+
+  end
+
+end
+
+end