chef-vpc-toolkit 2.1.0 → 2.2.0

data/CHANGELOG

@@ -1,3 +1,17 @@
+* Thu Feb 6 2011 Dan Prince <dan.prince@rackspace.com> - 2.2.0
+  - Support command execution with 'rake ssh'.
+  - The 'chef:tail_logs' task now supports the SERVER_NAME variable.
+  - Support for Ubuntu 10.10 maverick.
+  - Add support for connecting to servers which require SSL client auth.
+  - Implement 'group:add_server' task.
+  - Implement 'group:delete_server' task.
+  - Implement 'group:list REMOTE=true' task.
+  - Implement 'group:join' task.
+  - Implement 'group:join' task.
+  - Update the 'chef:install' task to support the SERVER_NAME variable.
+  - The 'chef:sync_repos' task is now 'chef:push_repos'.
+  - Implement a 'chef:pull_repos' task.
+
 * Tue Jan 11 2011 Dan Prince <dan.prince@rackspace.com> - 2.1.0
   - Implement 'rake vpn' tasks.
   - Set permission to 0600 on tmp/server_group files.

data/COPYING

@@ -1,7 +1,7 @@
 Unless otherwise noted, all files are released under the MIT license,
 exceptions contain licensing information in them.
 
-  Copyright (C) 2010 Rackspace US, Inc.
+  Copyright (C) 2010-2011 Rackspace US, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.rdoc

@@ -39,32 +39,54 @@ To install the gem:
 
 Example commands:
 
-	- Create a new cloud server group, upload cookbooks, install chef
+	- Create a new server group, upload cookbooks, install chef
 	on all the nodes, sync share data and cookbooks.
 
 		$ rake create
 
-	- List your currently running cloud server groups.
+	- List your currently running server groups.
 
 		$ rake group:list
 
-	- SSH into the current (most recently created) cloud server group
+	- SSH into the current (most recently created) server group
 
 		$ rake ssh
 
-	- SSH into a cloud server group with an ID of 3
+	- SSH into a server group with an ID of 3
 
 		$ rake ssh GROUP_ID=3
 
-	- Delete the cloud server group with an ID of 3
+	- Delete the server group with an ID of 3
 
 		$ rake group:delete GROUP_ID=3
 
-	- Rebuild/Re-Chef the 'db1' server in the most recently created cloud
-	server group
+	- Rebuild/Re-Chef the 'db1' server in the most recently created server group
 
 		$ rake rechef SERVER_NAME=db1
 
+	- Connect to a group as a VPN client. Currently supports and requires
+     Network Manager Openvpn, nmcli, and nm-applet (Linux only).
+
+		$ rake vpn
+
+	- Disconnect from group as a VPN client
+
+		$ rake vpn:disconnect
+
+== Bash Automation Script
+The following is an example bash script to spin up a group and run commands via SSH. Useful for CI automation in Hudson, etc.
+
+	#!/bin/bash
+
+	chef-vpc-toolkit -v
+	trap "rake group:delete" INT TERM EXIT # cleanup the group on exit
+
+	rake create
+	VPN_GW=$(rake group:show | grep "VPN gateway IP" | sed -e "s|VPN gateway IP: ||")
+	ssh root@$VPN_GW bash <<-EOF_BASH
+		# do stuff here
+	EOF_BASH
+
 == Author
 
 Dan Prince <dan.prince@rackspace.com>

data/VERSION

@@ -1 +1 @@
-2.1.0
+2.2.0

data/config/server_group.json

@@ -4,12 +4,12 @@
 	"description": "test description",
 	"servers": {
 		"login": {
-			"image_id": "51",
+			"image_id": "69",
 			"flavor_id": "2",
 			"openvpn_server": "true"
 		},
 		"client1": {
-			"image_id": "51",
+			"image_id": "69",
 			"flavor_id": "2"
 		}
 	}

data/lib/chef-vpc-toolkit.rb

@@ -1,7 +1,12 @@
 require 'chef-vpc-toolkit/util'
 require 'chef-vpc-toolkit/chef_installer'
-require 'chef-vpc-toolkit/cloud_servers_vpc'
-require 'chef-vpc-toolkit/http_util'
 require 'chef-vpc-toolkit/ssh_util'
 require 'chef-vpc-toolkit/version'
+require 'chef-vpc-toolkit/xml_util'
 require 'chef-vpc-toolkit/vpn_network_manager'
+require 'chef-vpc-toolkit/cloud-servers-vpc/connection'
+require 'chef-vpc-toolkit/cloud-servers-vpc/client'
+require 'chef-vpc-toolkit/cloud-servers-vpc/server'
+require 'chef-vpc-toolkit/cloud-servers-vpc/server_group'
+require 'chef-vpc-toolkit/cloud-servers-vpc/ssh_public_key'
+require 'chef-vpc-toolkit/cloud-servers-vpc/vpn_network_interface'

data/lib/chef-vpc-toolkit/chef-0.9.bash

@@ -30,6 +30,7 @@ fi
 
 sed -e "s|localhost|$SERVER_NAME|g" -i /etc/chef/client.rb
 sed -e "s|^chef_server_url.*|chef_server_url \"http://$SERVER_NAME:4000\"|g" -i /etc/chef/client.rb
+sed -e "s|^log_location.*|log_location \"\/var/log/chef/client.log\"|g" -i /etc/chef/client.rb
 
 local CHEF_CLIENT_CONF=/etc/default/chef-client
 [ -d /etc/sysconfig/ ] && CHEF_CLIENT_CONF=/etc/sysconfig/chef-client

data/lib/chef-vpc-toolkit/chef_bootstrap/fedora.bash

@@ -5,8 +5,8 @@ local INSTALL_TYPE=${1:-"CLIENT"} # CLIENT/SERVER
 	[[ "$INSTALL_TYPE" == "CLIENT" ]] || { echo "Chef server installations are not yet supported on Fedora."; exit 1; }
 
 	yum install -q -y ruby ruby-devel gcc gcc-c++ automake autoconf rubygems make &> /dev/null || { echo "Failed to install ruby, ruby-devel, etc."; exit 1; }
-	gem update --system
-	gem update
+	#gem update --system
+	#gem update
 	gem install json -v 1.1.4 --no-rdoc --no-ri &> /dev/null || \
 		{ echo "Failed to install JSON gem on $HOSTNAME."; exit 1; }
 	gem install ohai -v 0.5.6 --no-rdoc --no-ri &> /dev/null || \
@@ -34,7 +34,7 @@ validation_key         "/etc/chef/validation.pem"
 client_key             "/etc/chef/client.pem"
 EOF_CAT
 
-	CHEF_GEM_DIR=$(gem contents chef | sed -e "s|\(.*chef-0.9.8\).*|\1|" | head -n 1)
+	CHEF_GEM_DIR=$(gem contents chef 2>/dev/null | sed -e "s|\(.*chef-0.9.8\).*|\1|" | head -n 1)
 	cp $CHEF_GEM_DIR/distro/redhat/etc/init.d/chef-client /etc/init.d/
 	cp $CHEF_GEM_DIR/distro/redhat/etc/logrotate.d/chef-client /etc/logrotate.d/
 	chmod 755 /etc/init.d/chef-client

data/lib/chef-vpc-toolkit/chef_bootstrap/ubuntu.bash

@@ -1,6 +1,7 @@
 function install_chef {
 
 CODENAME=$(cat /etc/*release | grep CODENAME | sed -e "s|^.*=\([^$]*\)$|\1|")
+[[ "$CODENAME" == "maverick" ]] && CODENAME="lucid"
 local INSTALL_TYPE=${1:-"CLIENT"} # CLIENT/SERVER
 
 [ -f /etc/apt/sources.list.d/opscode.list ] || echo "deb http://apt.opscode.com $CODENAME main" > /etc/apt/sources.list.d/opscode.list

data/lib/chef-vpc-toolkit/chef_installer.rb

@@ -228,10 +228,17 @@ def self.tail_log(gateway_ip, server_name, log_file="/var/log/chef/client.log",
 	%x{ssh -o "StrictHostKeyChecking no" root@#{gateway_ip} ssh #{server_name} tail -n #{num_lines} #{log_file}}
 end
 
+
+def self.pull_cookbook_repos(options, local_dir="#{CHEF_VPC_PROJECT}/cookbook-repos/", remote_directory="/root/cookbook-repos")
+	$stdout.printf "Pulling remote Chef cookbook repositories..."
+	system("rsync -azL root@#{options['ssh_gateway_ip']}:#{remote_directory}/* '#{local_dir}'")
+	puts "OK"
+end
+
 def self.rsync_cookbook_repos(options, local_dir="#{CHEF_VPC_PROJECT}/cookbook-repos/", remote_directory="/root/cookbook-repos")
 
 	if File.exists?(local_dir) then
-		$stdout.printf "Syncing local Chef cookbook repositories..."
+		$stdout.printf "Pushing local Chef cookbook repositories..."
 		configs=Util.load_configs
 		%x{ssh -o "StrictHostKeyChecking no" root@#{options['ssh_gateway_ip']} bash <<-"EOF_SSH"
 			mkdir -p #{remote_directory}

data/lib/chef-vpc-toolkit/cloud-servers-vpc/client.rb

@@ -0,0 +1,186 @@
+module ChefVPCToolkit
+
+module CloudServersVPC
+
+class Client
+
+	@@data_dir=File.join(CHEF_VPC_PROJECT, "tmp", "clients")
+
+	def self.data_dir
+		@@data_dir
+	end
+
+	def self.data_dir=(dir)
+		@@data_dir=dir
+	end
+
+	attr_accessor :id
+	attr_accessor :name
+	attr_accessor :description
+	attr_accessor :status
+	attr_accessor :server_group_id
+
+	def initialize(options={})
+		@id=options[:id].to_i
+		@name=options[:name]
+		@description=options[:description]
+		@status=options[:status] or @status = "Pending"
+		@server_group_id=options[:server_group_id]
+
+		@vpn_network_interfaces=[]
+	end
+
+	def vpn_network_interfaces
+		@vpn_network_interfaces
+	end
+
+    def cache_to_disk
+        FileUtils.mkdir_p(@@data_dir)
+        File.open(File.join(@@data_dir, "#{@server_group_id}.xml"), 'w') do |f|
+            f.chmod(0600)
+            f.write(self.to_xml)
+        end
+    end
+
+	def delete
+		client_xml_file=File.join(@@data_dir, "#{@server_group_id}.xml")
+        if File.exists?(client_xml_file) then
+            File.delete(client_xml_file)
+        end
+	end
+
+	def self.from_xml(xml)
+
+		client=nil
+		dom = REXML::Document.new(xml)
+		REXML::XPath.each(dom, "/client") do |client|
+
+			client=Client.new(
+				:id => XMLUtil.element_text(client,"id").to_i,
+				:name => XMLUtil.element_text(client, "name"),
+				:description => XMLUtil.element_text(client,"description"),
+				:status => XMLUtil.element_text(client,"status"),
+				:server_group_id => XMLUtil.element_text(client, "server-group-id").to_i
+			)
+			REXML::XPath.each(dom, "//vpn-network-interface") do |vni|
+				vni = VpnNetworkInterface.new(
+					:id => XMLUtil.element_text(vni, "id"),
+					:vpn_ip_addr => XMLUtil.element_text(vni, "vpn-ip-addr"),
+					:ptp_ip_addr => XMLUtil.element_text(vni, "ptp-ip-addr"),
+					:client_key => XMLUtil.element_text(vni, "client-key"),
+					:client_cert => XMLUtil.element_text(vni, "client-cert"),
+					:ca_cert => XMLUtil.element_text(vni, "ca-cert")
+				)
+				client.vpn_network_interfaces << vni
+			end
+		end		 
+					
+		client
+					
+	end 
+
+    def to_xml
+
+        xml = Builder::XmlMarkup.new
+        xml.tag! "client" do |sg|
+            sg.id(@id)
+            sg.name(@name)
+            sg.description(@description)
+            sg.status(@status)
+            sg.tag! "server-group-id", @server_group_id
+            sg.tag! "vpn-network-interfaces", {"type" => "array"} do |interfaces|
+				@vpn_network_interfaces.each do |vni|
+					interfaces.tag! "vpn-network-interface" do |xml_vni|
+						xml_vni.id(vni.id)
+						xml_vni.tag! "vpn-ip-addr", vni.vpn_ip_addr
+						xml_vni.tag! "ptp-ip-addr", vni.ptp_ip_addr
+						xml_vni.tag! "client-key", vni.client_key
+						xml_vni.tag! "client-cert", vni.client_cert
+						xml_vni.tag! "ca-cert", vni.ca_cert
+					end	
+				end
+			end
+
+		end
+        xml.target!
+
+	end
+
+
+    # Poll the server group until it is online.
+    # :timeout - max number of seconds to wait before raising an exception.
+    #            Defaults to 1500
+    def poll_until_online(options={})
+
+        timeout=options[:timeout] or timeout = ENV['VPN_CLIENT_TIMEOUT']
+        if timeout.nil? or timeout.empty? then
+            timeout=300 # defaults to 5 minutes
+        end 
+
+		online = false
+		count=0
+		until online or (count*5) >= timeout.to_i do
+			count+=1
+			begin
+				client=Client.fetch(:id => @id, :source => "remote")
+
+				if client.status == "Online" then
+					online = true
+				else
+					yield client if block_given?
+					sleep 5
+				end
+			rescue EOFError
+			end
+		end
+		if (count*20) >= timeout.to_i then
+			raise "Timeout waiting for client to come online."
+		end
+
+	end
+
+	def self.create(server_group, client_name)
+
+		xml = Builder::XmlMarkup.new
+		xml.client do |client|
+			client.name(client_name)
+			client.description("Toolkit Client: #{client_name}")
+			client.tag! "server-group-id", server_group.id
+		end
+
+		xml=Connection.post("/clients.xml", xml.target!)
+		client=Client.from_xml(xml)
+		client.cache_to_disk
+		client
+
+	end
+
+    # Fetch a server group. The following options are available:
+    #
+    # :id - The ID of the group containing the client to fetch.
+    # :source - valid options are 'remote' and 'cache'
+    #
+	def self.fetch(options = {})
+
+        source = options[:source] or source = "remote"
+
+        if source == "remote" then
+			id=options[:id] or raise "Please specify a Client ID."
+			xml=Connection.get("/clients/#{id}.xml")
+			Client.from_xml(xml)
+		elsif source == "cache" then
+			id=options[:id] or id = ENV['GROUP_ID']
+			client_xml_file=File.join(@@data_dir, "#{id}.xml")
+			raise "No client files exist." if not File.exists?(client_xml_file)
+			Client.from_xml(IO.read(client_xml_file))
+		else
+			raise "Invalid fetch :source specified."
+		end
+
+	end
+
+end
+
+end
+
+end

data/lib/chef-vpc-toolkit/cloud-servers-vpc/connection.rb

@@ -0,0 +1,146 @@
+
+require 'uri'
+require 'net/http'
+require 'net/https'
+require 'openssl'
+require 'openssl/x509'
+
+module ChefVPCToolkit
+
+module CloudServersVPC
+
+class Connection
+
+MULTI_PART_BOUNDARY="jtZ!pZ1973um"
+
+	@@http=nil
+	@@auth_user=nil
+	@@auth_password=nil
+
+	def self.init_connection
+
+		configs=Util.load_configs
+
+		base_url = configs["cloud_servers_vpc_url"]
+		@@auth_user = configs["cloud_servers_vpc_username"]
+		@@auth_password = configs["cloud_servers_vpc_password"]
+
+		ssl_key = configs["ssl_key"]
+		ssl_cert = configs["ssl_cert"]
+		ssl_ca_cert = configs["ssl_ca_cert"]
+
+		url=URI.parse(base_url)
+		@@http = Net::HTTP.new(url.host,url.port)
+
+		if base_url =~ /^https/
+			@@http.use_ssl = true
+			if ssl_ca_cert then
+				@@http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+			else
+				@@http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+			end
+
+			if ssl_key then
+				pkey_data=IO.read(ssl_key)
+				if pkey_data.start_with?("-----BEGIN RSA PRIVATE KEY-----")
+					@@http.key=OpenSSL::PKey::RSA.new(pkey_data)
+				else
+					@@http.key=OpenSSL::PKey::DSA.new(pkey_data)
+				end
+			end
+			@@http.cert=OpenSSL::X509::Certificate.new(IO.read(ssl_cert)) if ssl_cert
+			@@http.ca_file=ssl_ca_cert if ssl_ca_cert
+		end
+
+	end
+
+	def self.file_upload(url_path, file_data={}, post_data={})
+		init_connection if @@http.nil?
+		req = Net::HTTP::Post.new(url_path)
+
+		post_arr=[]
+		post_data.each_pair do |key, value|
+			post_arr << "--#{MULTI_PART_BOUNDARY}\r\n"
+			post_arr << "Content-Disposition: form-data; name=\"#{key}\"\r\n"
+			post_arr << "\r\n"
+			post_arr << value
+			post_arr << "\r\n"
+		end
+
+		file_data.each_pair do |name, file|
+			post_arr << "--#{MULTI_PART_BOUNDARY}\r\n"
+			post_arr << "Content-Disposition: form-data; name=\"#{name}\"; filename=\"#{File.basename(file)}\"\r\n"
+			post_arr << "Content-Type: text/plain\r\n"
+			post_arr << "\r\n"
+			post_arr << File.read(file)
+			post_arr << "\r\n--#{MULTI_PART_BOUNDARY}--\r\n"
+		end
+		post_arr << "--#{MULTI_PART_BOUNDARY}--\r\n\r\n"
+
+		req.body=post_arr.join
+
+		req.basic_auth @@auth_user, @@auth_password if @@auth_user and @@auth_password
+		req["Content-Type"] = "multipart/form-data, boundary=#{MULTI_PART_BOUNDARY}"
+
+		response = @@http.request(req)
+		case response
+		when Net::HTTPSuccess
+			return response.body
+		else
+			puts response.body
+			response.error!
+		end
+	end
+
+	def self.post(url_path, post_data)
+		init_connection if @@http.nil?
+		req = Net::HTTP::Post.new(url_path)
+		if post_data.kind_of?(String) then
+			req.body=post_data
+		elsif post_data.kind_of?(Hash) then
+			req.form_data=post_data
+		else
+			raise "Invalid post data type."
+		end
+		req.basic_auth @@auth_user, @@auth_password if @@auth_user and @@auth_password
+		response = @@http.request(req)
+		case response
+		when Net::HTTPSuccess
+			return response.body
+		else
+			puts response.body
+			response.error!
+		end
+	end
+
+	def self.get(url_path)
+		init_connection if @@http.nil?
+		req = Net::HTTP::Get.new(url_path)
+		req.basic_auth @@auth_user, @@auth_password if @@auth_user and @@auth_password
+		response = @@http.request(req)
+		case response
+		when Net::HTTPSuccess
+			return response.body
+		else
+			response.error!
+		end
+	end
+
+	def self.delete(url_path)
+		init_connection if @@http.nil?
+		req = Net::HTTP::Delete.new(url_path)
+		req.basic_auth @@auth_user, @@auth_password if @@auth_user and @@auth_password
+		response = @@http.request(req)
+		case response
+		when Net::HTTPSuccess
+			return response.body
+		else
+			response.error!
+		end
+	end
+
+end
+
+end
+
+end