ironfan 3.1.0.rc1

data/lib/chef/knife/bootstrap/ubuntu11.10-ironfan.erb

@@ -0,0 +1,145 @@
+bash <<EOF || echo "Chef bootstrap failed!"
+
+# This is the ubuntu oneiric bootstrap script from infochimps' ironfan. It is
+# based on the opscode bootstrap script, with the important differences being it:
+#
+# * installs ruby 1.9.2 (not 1.8.7) using the system ruby
+# * upgrades rubygems rather than installing from source
+# * pushes the node identity into the first-boot.json
+# * installs the chef-client service and kicks off the first run of chef
+
+set -e
+
+<%= (@config[:verbosity].to_i > 1 ? 'set -v' : '') %>
+
+RUBY_VERSION=1.9.1
+CHEF_VERSION=<%= bootstrap_version_string.gsub(/.*[\s=]/,"") %>
+
+mkdir -p /tmp/knife-bootstrap
+chmod 700 /tmp/knife-bootstrap 
+cd /tmp/knife-bootstrap
+
+<%= "export http_proxy=\"#{knife_config[:bootstrap_proxy]}\"" if knife_config[:bootstrap_proxy] -%>
+eval `cat /etc/lsb-release `
+export DEBIAN_FRONTEND=noninteractive
+
+date > /etc/box_build_time
+
+# source for sun java if you want to install it later
+apt-get install -y python-software-properties
+add-apt-repository -y ppa:ferramroberto/java
+
+echo -e "`date` \n\n**** \n**** apt update:\n****\n"
+apt-get -y update
+apt-get -y upgrade
+
+echo -e "`date` \n\n**** \n**** Installing base packages:\n****\n"
+apt-get -y install build-essential make wget curl runit zlib1g-dev libssl-dev openssl libcurl4-openssl-dev libxml2-dev libxslt-dev libyaml-dev libreadline6 libreadline6-dev
+apt-get clean
+
+if [ ! -f /usr/bin/chef-client ]; then
+echo -e "`date` \n\n**** \n**** Installing ruby version ${RUBY_VERSION}:\n****\n"
+
+apt-get install -y ruby1.9.1 ruby1.9.1-dev
+
+if ruby -e "exit(%x{gem --version} < \"1.6.2\" ? 0 : -1 )" ; then
+  echo -e "`date` \n\n**** \n**** Updating rubygems:\n****\n"
+  # screw you Debian
+  REALLY_GEM_UPDATE_SYSTEM=1 gem update --system
+  # screw you rubygems
+  for foo in /usr/lib/ruby/site_ruby/*/rubygems/deprecate.rb ; do
+    # Don't have to be any such deprecations, in which case $foo won't exist
+    [ -f "$foo" ] && sudo sed -i.bak 's!@skip ||= false!true!' "$foo"
+  done
+fi
+
+echo -e "`date` \n\n**** \n**** Installing chef:\n****\n"
+gem install ohai --no-rdoc --no-ri
+gem install chef --no-rdoc --no-ri <%= bootstrap_version_string %>
+# gems needed for the client.rb or so generically useful you want them at hand
+gem install      --no-rdoc --no-ri extlib bundler json right_aws pry
+
+else # no chef-client
+echo -e "`date` \n\n**** \n**** Chef is present -- skipping apt/ruby/chef installation\n****\n"
+fi # end ruby+chef install
+
+# fix a bug in chef that prevents debugging template errors
+# will not work with --prerelease but that's OK hopefully opscode patches this crap soon
+bad_template_file="/usr/lib/ruby/gems/${RUBY_VERSION}/gems/chef-${CHEF_VERSION}/lib/chef/mixin/template.rb"
+if  echo "0505c482b8b0b333ac71bbc8a1795d19  $bad_template_file" | md5sum -c - 2>/dev/null ; then
+  curl https://github.com/mrflip/chef/commit/655a1967253a8759afb54f30b818bbcb7c309198.patch | sudo patch $bad_template_file
+fi
+
+echo -e "`date` \n\n**** \n**** Knifing in the chef client config files:\n****\n"
+mkdir -p /etc/chef
+
+<%- if @config[:client_key] %>
+(
+cat <<'EOP'
+<%= @config[:client_key] %>
+EOP
+) > /tmp/knife-bootstrap/client.pem
+awk NF /tmp/knife-bootstrap/client.pem > /etc/chef/client.pem
+<%- else %>
+(
+cat <<'EOP'
+<%= validation_key %>
+EOP
+) > /tmp/knife-bootstrap/validation.pem
+awk NF /tmp/knife-bootstrap/validation.pem > /etc/chef/validation.pem
+<%- end %>
+
+echo -e "`date` \n\n**** \n**** Nuking our temp files:\n****\n"
+
+cd /tmp
+rm -rf /tmp/knife-bootstrap
+
+echo -e "`date` \n\n**** \n**** Creating chef client script:\n****\n"
+
+(
+cat <<'EOP'
+<%= config_content %>
+<%= @config[:node].chef_client_script_content %>
+EOP
+) > /etc/chef/client.rb
+
+(
+cat <<'EOP'
+<%= { "run_list" => @run_list, "cluster_name" => @config[:node].cluster_name, "facet_name" => @config[:node].facet_name, "facet_index" => @config[:node].facet_index }.to_json %>
+EOP
+) > /etc/chef/first-boot.json
+
+echo -e "`date` \n\n**** \n**** Adding chef client runit scripts:\n****\n"
+( service chef-client stop >/dev/null 2>&1 ; sleep 1 ; killall chef-client 2>/dev/null ) || true
+mkdir -p /var/log/chef /var/chef /etc/service /etc/sv/chef-client/{log/main,supervise} 
+cat > /etc/sv/chef-client/log/run <<EOP
+#!/bin/bash
+exec svlogd -tt ./main
+EOP
+cat > /etc/sv/chef-client/run <<EOP
+#!/bin/bash
+exec 2>&1
+exec /usr/bin/env chef-client -i 43200 -s 20 -L /var/log/chef/client.log
+EOP
+chmod +x  /etc/sv/chef-client/log/run /etc/sv/chef-client/run
+ln -nfs /usr/bin/sv /etc/init.d/chef-client
+
+service chef-client stop >/dev/null 2>&1 || true
+
+<%- if (@config[:bootstrap_runs_chef_client].to_s == 'true') || (@chef_config.knife[:bootstrap_runs_chef_client].to_s == 'true') %>
+echo -e "`date` \n\n**** \n**** First run of chef:\n****\n"
+set -e
+<%= start_chef %>
+set +e
+<%- end %>
+
+echo -e "`date` \n\n**** \n**** Cleanup:\n****\n"
+# make locate work good
+updatedb
+
+echo -e "`date` \n\n**** \n**** Enabling chef client service:\n****\n"
+ln -nfs /etc/sv/chef-client /etc/service/chef-client
+service chef-client start
+
+echo -e "`date` \n\n**** \n**** Cluster Chef client bootstrap complete\n****\n"
+EOF

data/lib/chef/knife/cluster_bootstrap.rb

@@ -0,0 +1,74 @@
+#
+# Author:: Philip (flip) Kromer (<flip@infochimps.com>)
+# Copyright:: Copyright (c) 2011 Infochimps, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.dirname(__FILE__)+"/generic_command.rb")
+require 'chef/knife/bootstrap'
+
+class Chef
+  class Knife
+    class ClusterBootstrap < Ironfan::Script
+
+      option :ssh_user,
+        :long        => "--ssh-user USERNAME",
+        :short       => "-x USERNAME",
+        :description => "The ssh username"
+      option :bootstrap_runs_chef_client,
+        :long        => "--[no-]bootstrap-runs-chef-client",
+        :description => "If bootstrap is invoked, the bootstrap script causes an initial run of chef-client (default true).",
+        :boolean     => true,
+        :default     => true
+      option :distro,
+        :long        => "--distro DISTRO",
+        :short       => "-d DISTRO",
+        :description => "Bootstrap a distro using a template"
+      option :template_file,
+        :long        => "--template-file TEMPLATE",
+        :description => "Full path to location of template to use"
+      option :use_sudo,
+        :long        => "--sudo",
+        :description => "Execute the bootstrap via sudo",
+        :boolean     => true
+      option :host_key_verify,
+        :long => "--[no-]host-key-verify",
+        :description => "Verify host key, enabled by default.",
+        :boolean => true,
+        :default => true
+
+      import_banner_and_options(Chef::Knife::Bootstrap,
+        :except => [:chef_node_name, :run_list, :ssh_user, :distro, :template_file, :no_host_key_verify, :host_key_verify])
+      import_banner_and_options(Ironfan::Script)
+
+      deps do
+        Chef::Knife::Bootstrap.load_deps
+        Ironfan::Script.load_deps
+      end
+
+      def perform_execution(target)
+        target.each do |svr|
+          run_bootstrap(svr, svr.fog_server.dns_name)
+        end
+      end
+
+      def confirm_execution(target)
+        ui.info "Bootstrapping the node redoes its initial setup -- only do this on an aborted launch."
+        confirm_or_exit("Are you absolutely certain that you want to perform this action? (Type 'Yes' to confirm) ", 'Yes')
+      end
+
+    end
+  end
+end

data/lib/chef/knife/cluster_kick.rb

@@ -0,0 +1,94 @@
+#
+# Author:: Ash Berlin (ash_github@firemirror.om) and Philip (flip) Kromer (flip@infochimps.com)
+# Copyright:: Copyright (c) 2011 DigiResults Ltd. and Infochimps, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.dirname(__FILE__)+"/knife_common.rb")
+require File.expand_path(File.dirname(__FILE__)+"/cluster_ssh.rb")
+
+class Chef
+  class Knife
+    #
+    # Based on https://gist.github.com/1325982 by Ash Berlin
+    #
+    #     "Since chef v0.10 you can send USR1 to the chef-client process and it
+    #     will wake up and do a run. But the usual case when I want to do a run
+    #     is cos I'm either testing a cookbook change or I want to deploy now. I
+    #     could just run sudo chef-client but then that will only log to std
+    #     out.  Just run this script, it will send chef-client a USR1 signal and
+    #     then tail the log file (but nicely so that you'll get your terminal
+    #     back when the run completes)."
+    #
+    class ClusterKick < Chef::Knife::ClusterSsh
+
+      import_banner_and_options(Chef::Knife::ClusterSsh)
+      banner 'knife cluster kick "CLUSTER [FACET [INDEXES]]" (options) - start a run of chef-client on each server, tailing the logs and exiting when the run completes.'
+
+      option :pid_file,
+        :long        => "--pid_file",
+        :description => "Where to find the pid file. Typically /var/run/chef/client.pid (init.d) or /etc/sv/chef-client/supervise/pid (runit)",
+        :default     => "/etc/sv/chef-client/supervise/pid"
+
+      unless defined?(KICKSTART_SCRIPT)
+        KICKSTART_SCRIPT = <<EOF
+#!/bin/bash
+set -e
+<%= ((config[:verbosity].to_i > 1) ? "set -v" : "") %>
+
+if sudo service chef-client status ; then
+
+# running
+pid_file="<%= config[:pid_file] %>"
+log_file=/var/log/chef/client.log
+
+declare tail_pid
+
+on_exit() {
+  rm -f $pipe
+  [ -n "$tail_pid" ] && kill $tail_pid
+}
+
+trap "on_exit" EXIT ERR
+
+pipe=/tmp/pipe-$$
+mkfifo $pipe
+
+tail -fn0 "$log_file" > $pipe &
+
+tail_pid=$!
+
+sudo true
+pid="$(sudo cat $pid_file)"
+sudo kill -USR1 "$pid"
+sed -r "/(ERROR: Sleeping for [0-9]+ seconds before trying again|INFO: Report handlers complete)\$/{q}" $pipe
+
+else
+  echo -e "****\n\nchef-client daemon not running, invoking chef-client directly\n\n****\n"
+  sudo chef-client
+fi
+EOF
+      end
+
+      def run
+        @name_args = [ @name_args.join(' ') ]
+        script = Erubis::Eruby.new(KICKSTART_SCRIPT).result(:config => config)
+        @name_args[1] = script
+        super
+      end
+
+    end
+  end
+end

data/lib/chef/knife/cluster_kill.rb

@@ -0,0 +1,73 @@
+#
+# Author:: Chris Howe (<howech@infochimps.com>)
+# Copyright:: Copyright (c) 2011 Infochimps, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.dirname(__FILE__)+"/generic_command.rb")
+
+class Chef
+  class Knife
+    class ClusterKill < Ironfan::Script
+      import_banner_and_options(Ironfan::Script)
+
+      option :kill_bogus,
+        :long        => "--kill-bogus",
+        :description => "Kill bogus servers (ones that exist, but are not defined in the clusters file)",
+        :boolean     => true,
+        :default     => false
+      option :cloud,
+        :long        => '--[no-]cloud',
+        :description => "Kill machines from cloud (default is yes, terminate machines; use --no-cloud to skip)",
+        :boolean     => true,
+        :default     => true
+      option :chef,
+        :long        => "--[no-]chef",
+        :description => "Delete the chef node and client (default is yes, delete chef objects; use --no-chef to skip)",
+        :boolean     => true,
+        :default     => true
+
+      def relevant?(server)
+        server.killable?
+      end
+
+      # Execute every last mf'ing one of em
+      def perform_execution(target)
+        if config[:cloud]
+          section("Killing Cloud Machines")
+          target.select(&:in_cloud?).destroy
+        end
+
+        if config[:chef]
+          section("Killing Chef")
+          target.select(&:in_chef? ).delete_chef
+        end
+      end
+
+      def display(target, *args, &block)
+        super
+        ui.info Formatador.display_line("[red]Bogus servers detected[reset]: [blue]#{target.bogus_servers.map(&:fullname).inspect}[reset]") unless target.bogus_servers.empty?
+      end
+
+      def confirm_execution(target)
+        delete_message = [
+          (((!config[:chef])   || target.chef_nodes.empty?)  ? nil : "#{target.chef_nodes.length} chef nodes"),
+          (((!config[:cloud])  || target.fog_servers.empty?) ? nil : "#{target.fog_servers.length} fog servers") ].compact.join(" and ")
+        confirm_or_exit("Are you absolutely certain that you want to delete #{delete_message}? (Type 'Yes' to confirm) ", 'Yes')
+      end
+
+    end
+  end
+end

data/lib/chef/knife/cluster_launch.rb

@@ -0,0 +1,164 @@
+#
+# Author:: Philip (flip) Kromer (<flip@infochimps.com>)
+# Copyright:: Copyright (c) 2011 Infochimps, Inc
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.dirname(__FILE__)+"/knife_common.rb")
+require File.expand_path(File.dirname(__FILE__)+"/cluster_bootstrap.rb")
+
+class Chef
+  class Knife
+    class ClusterLaunch < Knife
+      include Ironfan::KnifeCommon
+
+      deps do
+        require 'time'
+        require 'socket'
+        Chef::Knife::ClusterBootstrap.load_deps
+      end
+
+      banner "knife cluster launch CLUSTER_NAME [FACET_NAME [INDEXES]] (options)"
+      [ :ssh_port, :ssh_password, :identity_file, :use_sudo,
+        :prerelease, :bootstrap_version, :template_file, :distro,
+        :bootstrap_runs_chef_client, :host_key_verify
+      ].each do |name|
+        option name, Chef::Knife::ClusterBootstrap.options[name]
+      end
+
+      option :dry_run,
+        :long        => "--dry-run",
+        :description => "Don't really run, just use mock calls",
+        :boolean     => true,
+        :default     => false
+      option :force,
+        :long        => "--force",
+        :description => "Perform launch operations even if it may not be safe to do so. Default false",
+        :boolean     => true,
+        :default     => false
+
+      option :bootstrap,
+        :long        => "--[no-]bootstrap",
+        :description => "Also bootstrap the launched node (default is NOT to bootstrap)",
+        :boolean     => true,
+        :default     => false
+
+      def run
+        load_ironfan
+        die(banner) if @name_args.empty?
+        configure_dry_run
+
+        #
+        # Load the facet
+        #
+        full_target = get_slice(*@name_args)
+        display(full_target)
+        target = full_target.select(&:launchable?)
+
+        warn_or_die_on_bogus_servers(full_target) unless full_target.bogus_servers.empty?
+
+        die("", "#{ui.color("All servers are running -- not launching any.",:blue)}", "", 1) if target.empty?
+
+        # Pre-populate information in chef
+        section("Sync'ing to chef and cloud")
+        target.sync_to_cloud
+        target.sync_to_chef
+
+        # Launch servers
+        section("Launching machines", :green)
+        target.create_servers
+
+        ui.info("")
+        display(target)
+
+        # As each server finishes, configure it
+        watcher_threads = target.parallelize do |svr|
+          perform_after_launch_tasks(svr)
+        end
+
+        progressbar_for_threads(watcher_threads)
+
+        display(target)
+      end
+
+      def display(target)
+        super(target, ["Name", "InstanceID", "State", "Flavor", "Image", "AZ", "Public IP", "Private IP", "Created At", 'Volumes', 'Elastic IP']) do |svr|
+          { 'launchable?' => (svr.launchable? ? "[blue]#{svr.launchable?}[reset]" : '-' ), }
+        end
+      end
+
+      def perform_after_launch_tasks(server)
+        # Wait for node creation on amazon side
+        server.fog_server.wait_for{ ready? }
+
+        # Try SSH
+        unless config[:dry_run]
+          nil until tcp_test_ssh(server.fog_server.dns_name){ sleep @initial_sleep_delay ||= 10  }
+        end
+
+        # Make sure our list of volumes is accurate
+        Ironfan.fetch_fog_volumes
+        server.discover_volumes!
+
+        # Attach volumes, etc
+        server.sync_to_cloud
+
+        # Run Bootstrap
+        if config[:bootstrap]
+          run_bootstrap(server, server.fog_server.dns_name)
+        end
+      end
+
+      def tcp_test_ssh(hostname)
+        tcp_socket = TCPSocket.new(hostname, 22)
+        readable = IO.select([tcp_socket], nil, nil, 5)
+        if readable
+          Chef::Log.debug("sshd accepting connections on #{hostname}, banner is #{tcp_socket.gets}")
+          yield
+          true
+        else
+          false
+        end
+      rescue Errno::ETIMEDOUT
+        false
+      rescue Errno::ECONNREFUSED
+        sleep 2
+        false
+      ensure
+        tcp_socket && tcp_socket.close
+      end
+
+      def warn_or_die_on_bogus_servers(target)
+        ui.info("")
+        ui.info "Cluster has servers in a transitional or undefined state (shown as 'bogus'):"
+        ui.info("")
+        display(target)
+        ui.info("")
+        unless config[:force]
+          die(
+            "Launch operations may be unpredictable under these circumstances.",
+            "You should wait for the cluster to stabilize, fix the undefined server problems",
+            "(run \"knife cluster show CLUSTER\" to see what the problems are), or launch",
+            "the cluster anyway using the --force option.", "", -2)
+        end
+        ui.info("")
+        ui.info "--force specified"
+        ui.info "Proceeding to launch anyway. This may produce undesired results."
+        ui.info("")
+      end
+
+    end
+  end
+end