chef-vpc-toolkit 2.7.2 → 2.8.0

data/CHANGELOG

@@ -1,3 +1,7 @@
+* Tue Nov 8 2011 Dan Prince <dan.prince@rackspace.com> - 2.8.0
+  - Add chef_interval option to chef_installer.conf.
+  - Add RESTART_ON_FAILURE and RESTART_TIMEOUT options to poll_clients task.
+
 * Sun Nov 6 2011 Dan Prince <dan.prince@rackspace.com> - 2.7.2
   - Support for configuring and starting Chef 0.10 when preinstalled.
 

data/VERSION

@@ -1 +1 @@
-2.7.2
+2.8.0

data/config/chef_installer.yml

@@ -18,3 +18,6 @@ chef_json_file: config/nodes.json
 
 # Chef server name
 chef_server_name: login
+
+# Chef client interval
+chef_interval: 600

data/lib/chef-vpc-toolkit/{chef-0.9.bash → chef_functions.bash}

@@ -1,6 +1,8 @@
 # Installation functions for Chef 0.8 RPMs obtained from the ELFF repo.
 export CHEF_STATUS_PORT="1234"
 export STATUS_MONITOR_DIR="/root/status_monitor"
+export SERVICE_BIN="/usr/sbin/service"
+[ -f /bin/rpm ] && SERVICE_BIN="/sbin/service"
 
 function configure_chef_server {
 
@@ -14,14 +16,15 @@ function print_client_validation_key {
 
 function configure_chef_client {
 
-if (( $# != 2 )); then
+if (( $# != 3 )); then
 	echo "Unable to configure chef client."
-	echo "usage: configure_chef_client <server_name> <client_validation_key>"
+	echo "usage: configure_chef_client <server_name> <client_validation_key> <interval>"
 	exit 1
 fi
 
 local SERVER_NAME=$1
 local CLIENT_VALIDATION_KEY=$2
+local INTERVAL=${3:-"600"} # default is 10 minutes
 
 if [ ! -f "/etc/chef/validation.pem" ]; then
 	cat > /etc/chef/validation.pem <<-EOF_VALIDATION_PEM
@@ -46,8 +49,8 @@ EOF_CAT_CHEF_CLIENT_CONF
 
 local CHEF_SYSCONFIG=/etc/default/chef-client
 [ -d /etc/sysconfig/ ] && CHEF_SYSCONFIG=/etc/sysconfig/chef-client
-cat > $CHEF_SYSCONFIG <<-"EOF_CAT_CHEF_SYSCONFIG"
-INTERVAL=600
+cat > $CHEF_SYSCONFIG <<-EOF_CAT_CHEF_SYSCONFIG
+INTERVAL=$INTERVAL
 SPLAY=20
 CONFIG=/etc/chef/client.rb
 LOGFILE=/var/log/chef/client.log
@@ -243,9 +246,6 @@ done
 
 function start_chef_server {
 
-	local SERVICE_BIN="/usr/sbin/service"
-	[ -f /bin/rpm ] && SERVICE_BIN="/sbin/service"
-
 	[ -d /var/run/chef ] && chown chef:chef /var/run/chef
 
 	if [ ! -f /var/run/chef/server.main.pid ]; then 
@@ -268,9 +268,6 @@ function start_chef_server {
 
 function start_chef_client {
 
-    local SERVICE_BIN="/usr/sbin/service"
-    [ -f /bin/rpm ] && SERVICE_BIN="/sbin/service"
-
 	$SERVICE_BIN chef-client start
     if [ -f /sbin/chkconfig ]; then
 		chkconfig chef-client on &> /dev/null
@@ -312,28 +309,49 @@ function poll_chef_client_online {
 
 local CLIENT_NAMES=${1:?"Please specify a chef client name."}
 local SECS=${2:-"600"} #10 minutes
+local RESTART_TIMEOUT=${3:-"$SECS"} #Restart clients if they haven't finished by timeout
+local RESTART_ON_FAILURE=${4} #Restart clients on failure (once)
+local TMP_RESTART=$(mktemp)
 
 local SLEEP_COUNT=5
 local COUNT=1
 local MAX_COUNT=$(( $SECS / $SLEEP_COUNT ))
+local MAX_RETRY_COUNT=$(( $RESTART_TIMEOUT / $SLEEP_COUNT ))
 local FAILED_CLIENTS=""
 local ALL_ONLINE="true"
 until (( $COUNT == $MAX_COUNT )); do
-    ALL_ONLINE="true"
-    FAILED_CLIENTS=""
-    for NAME in $CLIENT_NAMES; do
-            if ! grep "$NAME:" $STATUS_MONITOR_DIR/status.out | tail -n 1 | grep -c ":ONLINE" &> /dev/null; then
-                ALL_ONLINE="false"
-                FAILED_CLIENTS="$NAME $FAILED_CLIENTS"
-            fi
-    done
-    if [[ $ALL_ONLINE == "true" ]]; then
-        echo "All Chef client(s) ran successfully."
-        return 0
-    fi
-    COUNT=$(( $COUNT + 1 ))
-    sleep $SLEEP_COUNT
+	ALL_ONLINE="true"
+	FAILED_CLIENTS=""
+	for NAME in $CLIENT_NAMES; do
+			if ! grep "$NAME:" $STATUS_MONITOR_DIR/status.out | tail -n 1 | grep -c ":ONLINE" &> /dev/null; then
+				ALL_ONLINE="false"
+				FAILED_CLIENTS="$NAME $FAILED_CLIENTS"
+
+				#Restart any chef clients that might have failed (do this once)
+				if [ -n "$RESTART_ON_FAILURE" ] && grep "$NAME:" $STATUS_MONITOR_DIR/status.out &> /dev/null && ! grep ":$NAME:" $TMP_RESTART &> /dev/null; then
+					echo ":$NAME:" >> $TMP_RESTART
+					ssh "$NAME" bash <<-EOF_SSH_CHEF_RESTART
+						$SERVICE_BIN chef-client restart
+					EOF_SSH_CHEF_RESTART
+				fi
+
+				#Restart any chef clients that haven't completed by timeout
+				if (( $COUNT >= $MAX_RETRY_COUNT )) && ! grep "$NAME:" $STATUS_MONITOR_DIR/status.out &> /dev/null && ! grep ":$NAME:" $TMP_RESTART &> /dev/null; then
+					echo ":$NAME:" >> $TMP_RESTART
+					ssh "$NAME" bash <<-EOF_SSH_CHEF_RESTART
+						$SERVICE_BIN chef-client restart
+					EOF_SSH_CHEF_RESTART
+				fi
+			fi
+	done
+	if [[ $ALL_ONLINE == "true" ]]; then
+		echo "All Chef client(s) ran successfully."
+		return 0
+	fi
+	COUNT=$(( $COUNT + 1 ))
+	sleep $SLEEP_COUNT
 done
+[ -f "$TMP_RESTART" ] && rm $TMP_RESTART
 
 echo "Chef client(s) failed to run: $FAILED_CLIENTS"
 return 1

data/lib/chef-vpc-toolkit/chef_installer.rb

@@ -5,7 +5,7 @@ require 'yaml'
 module ChefVPCToolkit
 
 module ChefInstaller
-CHEF_INSTALL_FUNCTIONS=File.dirname(__FILE__) + "/chef-0.9.bash"
+CHEF_INSTALL_FUNCTIONS=File.dirname(__FILE__) + "/chef_functions.bash"
 
 def self.load_configs
 
@@ -74,7 +74,7 @@ json=JSON.parse(IO.read(options["chef_json_file"]))
 configure_client_script=""
 start_client_script=""
 if json.has_key?(options["chef_server_name"]) then
-	configure_client_script="configure_chef_client '#{options['chef_server_name']}' ''"
+	configure_client_script="configure_chef_client '#{options['chef_server_name']}' '' '#{options['chef_interval']}'"
 	start_client_script="start_chef_client"
 end
 knife_add_nodes_script=""
@@ -172,7 +172,7 @@ def self.install_chef_client(options, client_name, client_validation_key, os_typ
 	#{IO.read(File.dirname(__FILE__) + "/cloud_files.bash")}
 	#{IO.read(CHEF_INSTALL_FUNCTIONS)}
 	#{install_chef_script('CLIENT', os_type)}
-	configure_chef_client '#{options['chef_server_name']}' '#{client_validation_key}'
+	configure_chef_client '#{options['chef_server_name']}' '#{client_validation_key}' '#{options['chef_interval']}'
 	start_chef_client
 	EOF_BASH
 	EOF_GATEWAY
@@ -296,13 +296,13 @@ def self.rsync_cookbook_repos(options, local_dir="#{CHEF_VPC_PROJECT}/cookbook-r
 
 end
 
-def self.poll_clients(options, client_names, timeout=600)
+def self.poll_clients(options, client_names, timeout=600, restart_timeout=600, restart_once_on_failure="")
 
 output=%x{
 ssh -o "StrictHostKeyChecking no" root@#{options['ssh_gateway_ip']} bash <<-"EOF_GATEWAY"
 ssh #{options['chef_server_name']} bash <<-"EOF_BASH"
 #{IO.read(CHEF_INSTALL_FUNCTIONS)}
-poll_chef_client_online "#{client_names}" "#{timeout}"
+poll_chef_client_online "#{client_names}" "#{timeout}" "#{restart_timeout}" "#{restart_once_on_failure}"
 EOF_BASH
 EOF_GATEWAY
 }

data/rake/chef_vpc_toolkit.rake

@@ -219,6 +219,8 @@ namespace :chef do
 		
 		server_list=ENV['SERVER_NAME']
 		timeout=ENV['CHEF_TIMEOUT']
+		restart_timeout=ENV['CHEF_RESTART_TIMEOUT'] # restart if it takes this long
+		restart_on_failure=ENV['CHEF_RESTART_ON_FAILURE'] # restart once on failure
 		group=ServerGroup.fetch(:source => "cache")
 		if server_list.nil? or server_list.empty?
 		    server_list=group.server_names.collect{|x| x+" "}.join.to_s
@@ -229,7 +231,7 @@ namespace :chef do
 		configs=ChefInstaller.load_configs
 		configs["ssh_gateway_ip"]=group.vpn_gateway_ip
         puts "Polling for Chef clients to finish running..."
-        if not ChefInstaller.poll_clients(configs, server_list, timeout) then
+        if not ChefInstaller.poll_clients(configs, server_list, timeout, restart_timeout, restart_on_failure) then
 			raise "Chef client timeout."
 		end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: chef-vpc-toolkit
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 47
   prerelease: false
   segments: 
   - 2
-  - 7
-  - 2
-  version: 2.7.2
+  - 8
+  - 0
+  version: 2.8.0
 platform: ruby
 authors: 
 - Dan Prince
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-06 00:00:00 -04:00
+date: 2011-11-08 00:00:00 -05:00
 default_executable: chef-vpc-toolkit
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -115,7 +115,6 @@ files:
 - cookbook-repos/local/cookbooks/motd/templates/default/motd.erb
 - cookbook-repos/local/roles/README
 - lib/chef-vpc-toolkit.rb
-- lib/chef-vpc-toolkit/chef-0.9.bash
 - lib/chef-vpc-toolkit/chef_bootstrap/centos.bash
 - lib/chef-vpc-toolkit/chef_bootstrap/debian.bash
 - lib/chef-vpc-toolkit/chef_bootstrap/fedora.bash
@@ -123,6 +122,7 @@ files:
 - lib/chef-vpc-toolkit/chef_bootstrap/ubuntu-opscode.bash
 - lib/chef-vpc-toolkit/chef_bootstrap/ubuntu.bash
 - lib/chef-vpc-toolkit/chef_bootstrap/unknown.bash
+- lib/chef-vpc-toolkit/chef_functions.bash
 - lib/chef-vpc-toolkit/chef_installer.rb
 - lib/chef-vpc-toolkit/cloud-servers-vpc/client.rb
 - lib/chef-vpc-toolkit/cloud-servers-vpc/connection.rb