chef-vpc-toolkit 2.0.0

data/lib/chef-vpc-toolkit/chef_bootstrap/fedora.bash

@@ -0,0 +1,41 @@
+function install_chef {
+
+local INSTALL_TYPE=${1:-"CLIENT"} # CLIENT/SERVER
+
+	[[ "$INSTALL_TYPE" == "CLIENT" ]] || { echo "Chef server installations are not yet supported on Fedora."; exit 1; }
+
+	yum install -q -y ruby ruby-devel gcc gcc-c++ automake autoconf rubygems make &> /dev/null || { echo "Failed to install ruby, ruby-devel, etc."; exit 1; }
+	gem update --system
+	gem update
+	gem install json -v 1.1.4 --no-rdoc --no-ri &> /dev/null || \
+		{ echo "Failed to install JSON gem on $HOSTNAME."; exit 1; }
+	gem install ohai -v 0.5.6 --no-rdoc --no-ri &> /dev/null || \
+		{ echo "Failed to install ohai gem on $HOSTNAME."; exit 1; }
+	gem install chef -v 0.9.8 --no-rdoc --no-ri &> /dev/null || \
+		{ echo "Failed to install chef gem on $HOSTNAME."; exit 1; }
+
+	for DIR in /etc/chef /var/log/chef /var/cache/chef /var/lib/chef /var/run/chef; do
+		mkdir -p $DIR
+	done
+
+	cat > /etc/chef/client.rb <<-"EOF_CAT"
+log_level          :info
+log_location       STDOUT
+ssl_verify_mode    :verify_none
+chef_server_url "http://localhost:4000"
+file_cache_path    "/var/cache/chef"
+file_backup_path   "/var/lib/chef/backup"
+pid_file           "/var/run/chef/client.pid"
+cache_options({ :path => "/var/cache/chef/checksums", :skip_expires => true})
+signing_ca_user "chef"
+Mixlib::Log::Formatter.show_time = true
+validation_client_name "chef-validator"
+validation_key         "/etc/chef/validation.pem"
+client_key             "/etc/chef/client.pem"
+EOF_CAT
+
+	cp /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/distro/redhat/etc/init.d/chef-client /etc/init.d/
+	cp /usr/lib/ruby/gems/1.8/gems/chef-0.9.8/distro/redhat/etc/logrotate.d/chef-client /etc/logrotate.d/
+	chmod 755 /etc/init.d/chef-client
+
+}

data/lib/chef-vpc-toolkit/chef_bootstrap/rhel.bash

@@ -0,0 +1,38 @@
+function install_chef {
+
+local INSTALL_TYPE=${1:-"CLIENT"} # CLIENT/SERVER
+
+# cached RPMs from ELFF
+local CDN_BASE="http://c2521002.cdn.cloudfiles.rackspacecloud.com"
+
+local TARBALL="chef-client-0.9.8-rhel5-x86_64.tar.gz"
+if [[ "$INSTALL_TYPE" == "SERVER" ]]; then
+  TARBALL="chef-server-0.9.8-rhel5-x86_64.tar.gz"
+fi
+
+rpm -q rsync &> /dev/null || yum install -y -q rsync
+rpm -q wget &> /dev/null || yum install -y -q wget
+
+if ! rpm -q rubygem-chef &> /dev/null; then
+
+	local CHEF_RPM_DIR=$(mktemp -d)
+
+	wget "$CDN_BASE/$TARBALL" -O "$CHEF_RPM_DIR/chef.tar.gz" &> /dev/null \
+		|| { echo "Failed to download Chef RPM tarball."; exit 1; }
+	cd $CHEF_RPM_DIR
+
+	tar xzf chef.tar.gz || { echo "Failed to extract Chef tarball."; exit 1; }
+	rm chef.tar.gz
+	cd chef*
+	yum install -q -y --nogpgcheck */*.rpm
+	if [[ "$INSTALL_TYPE" == "SERVER" ]]; then
+		rpm -q rubygem-chef-server &> /dev/null || { echo "Failed to install chef."; exit 1; }
+	else
+		rpm -q rubygem-chef &> /dev/null || { echo "Failed to install chef."; exit 1; }
+	fi
+	cd /tmp
+	rm -Rf "$CHEF_RPM_DIR"
+
+fi
+
+}

data/lib/chef-vpc-toolkit/chef_bootstrap/ubuntu.bash

@@ -0,0 +1,32 @@
+function install_chef {
+
+CODENAME=$(cat /etc/*release | grep CODENAME | sed -e "s|^.*=\([^$]*\)$|\1|")
+local INSTALL_TYPE=${1:-"CLIENT"} # CLIENT/SERVER
+
+[ -f /etc/apt/sources.list.d/opscode.list ] || echo "deb http://apt.opscode.com $CODENAME main" > /etc/apt/sources.list.d/opscode.list
+wget -q -O- http://apt.opscode.com/packages@opscode.com.gpg.key | apt-key add - &> /dev/null || { echo "Failed to configure Apt repo."; exit 1; }
+
+dpkg -L rsync &> /dev/null || apt-get install -y rsync &> /dev/null
+
+if ! dpkg -L chef &> /dev/null; then
+
+	if [[ "$INSTALL_TYPE" == "SERVER" ]]; then
+
+		[[ "$CODENAME" == "lucid" ]] || { echo "Ubuntu 10.0.4 lucid is required for Chef server installations."; exit 1; }
+		apt-get update &> /dev/null || { echo "Failed to apt-get update."; exit 1; }
+		echo "chef-solr    chef-solr/amqp_password    password  YA1B2C301234Z" | debconf-set-selections &> /dev/null || { echo "Failed to set debconf selections for chef-solr."; exit 1; }
+		echo "chef    chef/chef_server_url    string  http://localhost:4000" | debconf-set-selections &> /dev/null || { echo "Failed to set debconf selections for chef."; exit 1; }
+		DEBIAN_FRONTEND=noninteractive apt-get install -y chef-server chef &> /dev/null || { echo "Failed to install the Chef Server via apt-get on $HOSTNAME."; exit 1; }
+	else
+		apt-get update &> /dev/null || { echo "Failed to apt-get update."; exit 1; }
+		echo "chef    chef/chef_server_url    string  http://localhost:4000" | debconf-set-selections &> /dev/null || { echo "Failed to set debconf selections for chef."; exit 1; }
+		DEBIAN_FRONTEND=noninteractive apt-get install -y chef &> /dev/null || { echo "Failed to install Chef via apt-get on $HOSTNAME."; exit 1; }
+	fi
+
+	/etc/init.d/chef-client stop &> /dev/null
+	sleep 2
+	rm /var/log/chef/client.log
+
+fi
+
+}

data/lib/chef-vpc-toolkit/chef_installer.rb

@@ -0,0 +1,276 @@
+require 'rubygems'
+require 'json'
+require 'yaml'
+
+module ChefVPCToolkit
+
+module ChefInstaller
+CHEF_INSTALL_FUNCTIONS=File.dirname(__FILE__) + "/chef-0.9.bash"
+
+def self.load_configs
+
+	config_file=CHEF_VPC_PROJECT + File::SEPARATOR + "config" + File::SEPARATOR + "chef_installer.yml"
+
+	if File.exists?(config_file) then
+		return YAML.load_file(config_file)
+	else
+		raise "The config/chef_installer.conf file is missing."
+	end
+
+end
+
+# validate the chef.json config file by parsing it
+def self.validate_json(options)
+
+	Util.raise_if_nil_or_empty(options, "chef_json_file")
+	begin
+		JSON.parse(IO.read(options["chef_json_file"]))
+	rescue Exception => e
+		puts "Failed to parse Chef JSON config file:"
+		puts ""
+		raise		
+	end
+
+	if not options["databags_json_file"].nil? and not options["databags_json_file"].empty?
+		begin
+			JSON.parse(IO.read(options["databags_json_file"]))
+		rescue Exception => e
+			puts "Failed to parse Databag JSON config file:"
+			puts ""
+			raise		
+		end
+	end
+
+end
+
+def self.install_chef_server(options, machine_os_types)
+
+Util.raise_if_nil_or_empty(options, "ssh_gateway_ip")
+Util.raise_if_nil_or_empty(options, "chef_json_file")
+Util.raise_if_nil_or_empty(options, "chef_server_name")
+
+# should we install a Chef client on the server?
+json=JSON.parse(IO.read(options["chef_json_file"]))
+configure_client_script=""
+start_client_script=""
+if json.has_key?(options["chef_server_name"]) then
+	configure_client_script="configure_chef_client '#{options['chef_server_name']}' ''"
+	start_client_script="start_chef_client"
+end
+knife_add_nodes_script=""
+json.each_pair do |node_name, node_json|
+	run_list=node_json['run_list'].inspect
+	node_json.delete("run_list")
+	attributes=node_json.to_json.to_s
+	knife_add_nodes_script+="knife_add_node '#{node_name}' '#{run_list}' '#{attributes}'\n"
+end
+
+cookbook_urls=""
+if options["chef_cookbook_repos"] then
+	cookbook_urls=options["chef_cookbook_repos"].inject { |sum, c| sum + " " + c }
+end
+os_type=machine_os_types[options['chef_server_name']]
+
+data=%x{
+ssh root@#{options['ssh_gateway_ip']} bash <<-"EOF_GATEWAY"
+ssh #{options['chef_server_name']} bash <<-"EOF_BASH"
+echo "Installing Chef server on: $HOSTNAME"
+EOF_BASH
+EOF_GATEWAY
+}
+puts data
+
+data=%x{
+ssh root@#{options['ssh_gateway_ip']} bash <<-"EOF_GATEWAY"
+ssh #{options['chef_server_name']} bash <<-"EOF_BASH"
+#{IO.read(File.dirname(__FILE__) + "/cloud_files.bash")}
+#{IO.read(File.dirname(__FILE__) + "/chef_bootstrap/#{os_type}.bash")}
+#{IO.read(CHEF_INSTALL_FUNCTIONS)}
+install_chef "SERVER"
+
+mkdir -p /root/cookbook-repos
+
+configure_chef_server
+start_chef_server
+
+#{configure_client_script}
+
+configure_knife "#{options["knife_editor"]}"
+
+knife_upload_cookbooks_and_roles
+
+#{knife_add_nodes_script}
+
+#{start_client_script}
+
+EOF_BASH
+EOF_GATEWAY
+}
+puts data
+
+return client_validation_key(options)
+
+end
+
+def self.client_validation_key(options)
+
+client_validation_key=%x{
+ssh root@#{options['ssh_gateway_ip']} bash <<-"EOF_GATEWAY"
+ssh #{options['chef_server_name']} bash <<-"EOF_BASH"
+#{IO.read(CHEF_INSTALL_FUNCTIONS)}
+print_client_validation_key
+EOF_BASH
+EOF_GATEWAY
+}
+
+raise "Client validation key is blank." if client_validation_key.nil? or client_validation_key.empty?
+
+return client_validation_key
+
+end
+
+def self.install_chef_clients(options, client_validation_key, os_types)
+
+	# configure Chef clients on each node
+	json=JSON.parse(IO.read(options['chef_json_file']))
+	json.each_pair do |hostname, json_hash|
+		if hostname != options['chef_server_name']
+			install_chef_client(options, hostname, client_validation_key, os_types[hostname])
+		end
+	end
+
+end
+
+def self.install_chef_client(options, client_name, client_validation_key, os_type)
+
+	puts "Installing Chef client on: #{client_name}"
+
+	data=%x{
+	ssh root@#{options['ssh_gateway_ip']} bash <<-"EOF_GATEWAY"
+	ssh #{client_name} bash <<-"EOF_BASH"
+	#{IO.read(File.dirname(__FILE__) + "/cloud_files.bash")}
+	#{IO.read(File.dirname(__FILE__) + "/chef_bootstrap/#{os_type}.bash")}
+	#{IO.read(CHEF_INSTALL_FUNCTIONS)}
+	install_chef "CLIENT"
+	configure_chef_client '#{options['chef_server_name']}' '#{client_validation_key}'
+	start_chef_client
+	EOF_BASH
+	EOF_GATEWAY
+	}
+	puts data
+
+end
+
+def self.create_databags(options)
+
+Util.raise_if_nil_or_empty(options, "ssh_gateway_ip")
+
+if options["databags_json_file"].nil? or options["databags_json_file"].empty?
+puts "No databag config file specified."
+return
+end
+printf "Creating databags..."
+STDOUT.flush
+
+if not File.exists?(options["databags_json_file"]) then
+	raise "Databags json file is missing: #{options["databags_json_file"]}."
+end
+
+json=JSON.parse(IO.read(options["databags_json_file"]))
+
+databag_cmds=""
+
+json.each_pair do |bag_name, items_json|
+	databag_cmds+="knife data bag delete '#{bag_name}' -y &> /dev/null \n"
+	databag_cmds+="knife data bag create '#{bag_name}' -y \n"
+
+	items_json.each do |item_json|
+
+	item_id=item_json["id"]
+	raise "Databags json missing item ID." if item_id.nil? or item_id.empty?
+	databag_cmds+="knife_create_databag '#{bag_name}' '#{item_id}' '#{item_json.to_json.to_s}'\n"
+	end
+
+data=%x{
+ssh root@#{options['ssh_gateway_ip']} bash <<-"EOF_GATEWAY"
+#{IO.read(CHEF_INSTALL_FUNCTIONS)}
+#{databag_cmds}
+EOF_GATEWAY
+}
+puts "OK."
+
+end
+
+end
+
+def self.knife_readd_node(options, client_name)
+
+Util.raise_if_nil_or_empty(options, "ssh_gateway_ip")
+Util.raise_if_nil_or_empty(options, "chef_json_file")
+
+json=JSON.parse(IO.read(options["chef_json_file"]))
+node_json=json[client_name]
+run_list=node_json['run_list'].inspect
+node_json.delete("run_list")
+attributes=node_json.to_json.to_s
+data=%x{
+ssh root@#{options['ssh_gateway_ip']} bash <<-"EOF_GATEWAY"
+#{IO.read(CHEF_INSTALL_FUNCTIONS)}
+knife_delete_node '#{client_name}'
+knife_add_node '#{client_name}' '#{run_list}' '#{attributes}'
+EOF_GATEWAY
+}
+
+end
+
+def self.tail_log(gateway_ip, server_name, log_file="/var/log/chef/client.log", num_lines="100")
+	%x{ssh root@#{gateway_ip} ssh #{server_name} tail -n #{num_lines} #{log_file}}
+end
+
+def self.rsync_cookbook_repos(options, local_dir="#{CHEF_VPC_PROJECT}/cookbook-repos/", remote_directory="/root/cookbook-repos")
+
+	if File.exists?(local_dir) then
+		$stdout.printf "Syncing local Chef cookbook repositories..."
+		configs=Util.load_configs
+		%x{ssh root@#{options['ssh_gateway_ip']} bash <<-"EOF_SSH"
+			mkdir -p #{remote_directory}
+			if [ -f /usr/bin/yum ]; then
+				rpm -q rsync &> /dev/null || yum install -y -q rsync
+			else
+				dpkg -L rsync > /dev/null 2>&1 || apt-get install -y --quiet rsync > /dev/null 2>&1
+			fi
+		EOF_SSH
+		}
+		system("rsync -azL '#{local_dir}' root@#{options['ssh_gateway_ip']}:#{remote_directory}")
+		puts "OK"
+	end
+
+	cookbook_urls=""
+	if options["chef_cookbook_repos"] then
+		cookbook_urls=options["chef_cookbook_repos"].inject { |sum, c| sum + " " + c }
+	end
+
+	data=%x{
+	ssh root@#{options['ssh_gateway_ip']} bash <<-"EOF_SSH"
+	#{IO.read(File.dirname(__FILE__) + "/cloud_files.bash")}
+	#{IO.read(CHEF_INSTALL_FUNCTIONS)}
+
+	if [ -n "#{cookbook_urls}" ]; then
+		download_cookbook_repos "#{cookbook_urls}"
+	fi
+
+	if [ -f /root/.chef/knife.rb ]; then
+		echo -n "Uploading cookbooks and roles..."
+		knife_upload_cookbooks_and_roles
+		echo "OK"
+	fi
+
+	EOF_SSH
+	}
+	puts data
+
+end
+
+end
+
+end

data/lib/chef-vpc-toolkit/cloud_files.bash

@@ -0,0 +1,67 @@
+# Load the username and password into variables that are used by this
+# bash API.
+function install_curl {
+
+if [ -f /usr/bin/yum ]; then
+	rpm -q curl &> /dev/null || yum install -y -q curl
+elif [ -f /usr/bin/dpkg ]; then
+	dpkg -L curl > /dev/null 2>&1 || apt-get install -y --quiet curl > /dev/null 2>&1
+else
+	echo "Failed to install curl. Unsupported platform."
+	exit 1
+fi
+
+}
+
+function load_cloud_configs {
+
+	if [ -z "$RACKSPACE_CLOUD_API_USERNAME" ] || [ -z "$RACKSPACE_CLOUD_API_KEY" ]; then
+
+		if [ ! -f ~/.rackspace_cloud ]; then
+			echo "Missing .rackspace_cloud config file."
+			exit 1
+		fi
+
+		export RACKSPACE_CLOUD_API_USERNAME=$(cat ~/.rackspace_cloud | grep "userid" | sed -e "s|.*: \([^ \n\r]*\).*|\1|")
+		export RACKSPACE_CLOUD_API_KEY=$(cat ~/.rackspace_cloud | grep "api_key" | sed -e "s|.*: \([^ \n\r]*\).*|\1|")
+
+		if [ -z "$RACKSPACE_CLOUD_API_USERNAME" ] || [ -z "$RACKSPACE_CLOUD_API_KEY" ]; then
+			echo "Please define a 'userid' and 'api_key' in ~/.rackspace_cloud"
+			exit 1
+		fi
+
+	fi
+
+}
+
+# Download a private file from Rackspace Cloud Files using the secure
+# X-Storage-Url.
+function download_cloud_file {
+
+	if (( $# != 2 )); then
+		echo "Failed to download cloud file."
+		echo "usage: download_cloud_file <container_url> <output_file>"
+		exit 1
+	fi
+
+	load_cloud_configs
+	install_curl
+
+	local CONTAINER_URL=$1
+	local OUTPUT_FILE=$2
+
+	local AUTH_RESPONSE=$(curl -D - \
+		-H "X-Auth-Key: $RACKSPACE_CLOUD_API_KEY" \
+		-H "X-Auth-User: $RACKSPACE_CLOUD_API_USERNAME" \
+		"https://auth.api.rackspacecloud.com/v1.0" 2> /dev/null)
+
+	[[ $? == 0 ]] || { echo "Failed to authenticate."; exit 1; }
+
+	local AUTH_TOKEN=$(echo $AUTH_RESPONSE | \
+		sed -e "s|.* X-Auth-Token: \([^ \n\r]*\).*|\1|g")
+	local STORAGE_URL=$(echo $AUTH_RESPONSE | \
+		sed -e "s|.* X-Storage-Url: \([^ \n\r]*\).*|\1|g")
+
+	curl -s -X GET -H "X-Auth-Token: $AUTH_TOKEN" "$STORAGE_URL/$CONTAINER_URL" -o "$OUTPUT_FILE"
+
+}