cluster_chef-knife 3.0.12 → 3.0.14

data/ironfan.gemspec

@@ -0,0 +1,123 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "ironfan"
+  s.version = "3.0.14"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Infochimps"]
+  s.date = "2012-02-15"
+  s.description = "ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
+  s.email = "coders@infochimps.com"
+  s.extra_rdoc_files = [
+    "LICENSE",
+    "README.md"
+  ]
+  s.files = [
+    ".gitignore",
+    ".rspec",
+    "CHANGELOG.md",
+    "Gemfile",
+    "LICENSE",
+    "README.md",
+    "Rakefile",
+    "TODO.md",
+    "VERSION",
+    "chefignore",
+    "cluster_chef-knife.gemspec",
+    "cluster_chef.gemspec",
+    "config/client.rb",
+    "config/proxy.pac",
+    "config/ubuntu10.04-cluster_chef.erb",
+    "config/ubuntu11.10-cluster_chef.erb",
+    "lib/chef/knife/bootstrap/ubuntu10.04-cluster_chef.erb",
+    "lib/chef/knife/bootstrap/ubuntu11.10-cluster_chef.erb",
+    "lib/chef/knife/cluster_bootstrap.rb",
+    "lib/chef/knife/cluster_kick.rb",
+    "lib/chef/knife/cluster_kill.rb",
+    "lib/chef/knife/cluster_launch.rb",
+    "lib/chef/knife/cluster_list.rb",
+    "lib/chef/knife/cluster_proxy.rb",
+    "lib/chef/knife/cluster_show.rb",
+    "lib/chef/knife/cluster_ssh.rb",
+    "lib/chef/knife/cluster_start.rb",
+    "lib/chef/knife/cluster_stop.rb",
+    "lib/chef/knife/cluster_sync.rb",
+    "lib/chef/knife/generic_command.rb",
+    "lib/chef/knife/knife_common.rb",
+    "lib/cluster_chef.rb",
+    "lib/cluster_chef/chef_layer.rb",
+    "lib/cluster_chef/cloud.rb",
+    "lib/cluster_chef/cluster.rb",
+    "lib/cluster_chef/compute.rb",
+    "lib/cluster_chef/deprecated.rb",
+    "lib/cluster_chef/discovery.rb",
+    "lib/cluster_chef/dsl_object.rb",
+    "lib/cluster_chef/facet.rb",
+    "lib/cluster_chef/fog_layer.rb",
+    "lib/cluster_chef/private_key.rb",
+    "lib/cluster_chef/role_implications.rb",
+    "lib/cluster_chef/security_group.rb",
+    "lib/cluster_chef/server.rb",
+    "lib/cluster_chef/server_slice.rb",
+    "lib/cluster_chef/volume.rb",
+    "notes/aws_console_screenshot.jpg",
+    "rspec.watchr",
+    "spec/cluster_chef/cluster_spec.rb",
+    "spec/cluster_chef/facet_spec.rb",
+    "spec/cluster_chef/server_slice_spec.rb",
+    "spec/cluster_chef/server_spec.rb",
+    "spec/cluster_chef_spec.rb",
+    "spec/spec_helper.rb",
+    "spec/spec_helper/dummy_chef.rb",
+    "spec/test_config.rb",
+    "tasks/chef_config.rake",
+    "tasks/jeweler_use_alt_branch.rake"
+  ]
+  s.homepage = "http://infochimps.com/labs"
+  s.licenses = ["apachev2"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.15"
+  s.summary = "ironfan allows you to orchestrate not just systems but clusters of machines. It includes a powerful layer on top of knife and a collection of cloud cookbooks."
+  s.test_files = ["spec/cluster_chef/cluster_spec.rb", "spec/cluster_chef/facet_spec.rb", "spec/cluster_chef/server_slice_spec.rb", "spec/cluster_chef/server_spec.rb", "spec/cluster_chef_spec.rb", "spec/spec_helper/dummy_chef.rb", "spec/spec_helper.rb", "spec/test_config.rb"]
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<chef>, ["~> 0.10.4"])
+      s.add_runtime_dependency(%q<fog>, ["~> 1.1.1"])
+      s.add_runtime_dependency(%q<formatador>, ["~> 0.2.1"])
+      s.add_runtime_dependency(%q<gorillib>, ["~> 0.1.7"])
+      s.add_development_dependency(%q<bundler>, ["~> 1"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.5"])
+      s.add_development_dependency(%q<yard>, ["~> 0.6"])
+      s.add_development_dependency(%q<configliere>, ["~> 0.4.8"])
+    else
+      s.add_dependency(%q<chef>, ["~> 0.10.4"])
+      s.add_dependency(%q<fog>, ["~> 1.1.1"])
+      s.add_dependency(%q<formatador>, ["~> 0.2.1"])
+      s.add_dependency(%q<gorillib>, ["~> 0.1.7"])
+      s.add_dependency(%q<bundler>, ["~> 1"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6"])
+      s.add_dependency(%q<rspec>, ["~> 2.5"])
+      s.add_dependency(%q<yard>, ["~> 0.6"])
+      s.add_dependency(%q<configliere>, ["~> 0.4.8"])
+    end
+  else
+    s.add_dependency(%q<chef>, ["~> 0.10.4"])
+    s.add_dependency(%q<fog>, ["~> 1.1.1"])
+    s.add_dependency(%q<formatador>, ["~> 0.2.1"])
+    s.add_dependency(%q<gorillib>, ["~> 0.1.7"])
+    s.add_dependency(%q<bundler>, ["~> 1"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6"])
+    s.add_dependency(%q<rspec>, ["~> 2.5"])
+    s.add_dependency(%q<yard>, ["~> 0.6"])
+    s.add_dependency(%q<configliere>, ["~> 0.4.8"])
+  end
+end
+

data/lib/chef/knife/bootstrap/ubuntu10.04-cluster_chef.erb

@@ -1,30 +1,40 @@
-bash -c '
+bash <<EOF || echo "Chef bootstrap failed!"
 
-# This is the ubuntu-10.04-gems script from opscode, but it
-# * installs ruby 1.9.2, not 1.8.7
+# This is the ubuntu natty bootstrap script from infochimps' cluster_chef. It is
+# based on opscode's bootstrap script, with the following important differences: 
+#
+# * installs ruby 1.9.2 (not 1.8.7) from source
 # * upgrades rubygems rather than installing from source
 # * pushes the node identity into the first-boot.json
 # * installs the chef-client service and kicks off the first run of chef
 
-<%= (@config[:verbosity].to_i > 1 ? 'set -v' : '') %>
+set -e
 
-mkdir -p /tmp/knife-bootstrap ; chmod 700 /tmp/knife-bootstrap 
-cd /tmp/knife-bootstrap
+<%= (@config[:verbosity].to_i > 1 ? 'set -v' : '') %>
 
 RUBY_VERSION=1.9.2-p290
+CHEF_VERSION=<%= bootstrap_version_string.gsub(/.*[\s=]/,"") %>
+
+mkdir -p /tmp/knife-bootstrap
+chmod 700 /tmp/knife-bootstrap 
+cd /tmp/knife-bootstrap
 
 <%= "export http_proxy=\"#{knife_config[:bootstrap_proxy]}\"" if knife_config[:bootstrap_proxy] -%>
 eval `cat /etc/lsb-release `
 export DEBIAN_FRONTEND=noninteractive
 
-if [ ! -f /usr/bin/chef-client ]; then
+date > /etc/box_build_time
+
 echo -e "`date` \n\n**** \n**** apt update:\n****\n"
-apt-get    update
+apt-get -y update
 apt-get -y upgrade
 
 echo -e "`date` \n\n**** \n**** Installing base packages:\n****\n"
-apt-get install -y build-essential wget runit runit-services zlib1g-dev libssl-dev openssl libcurl4-openssl-dev libreadline6-dev libyaml-dev
+apt-get -y install build-essential make wget curl runit zlib1g-dev libssl-dev openssl libcurl4-openssl-dev libxml2-dev libxslt-dev libyaml-dev libreadline6 libreadline6-dev
+apt-get -y install runit-services
+apt-get clean
 
+if [ ! -f /usr/bin/chef-client ]; then
 echo -e "`date` \n\n**** \n**** Installing ruby version ${RUBY_VERSION}:\n****\n"
 
 wget ftp://ftp.ruby-lang.org//pub/ruby/1.9/ruby-${RUBY_VERSION}.tar.gz
@@ -46,21 +56,28 @@ update-alternatives \
 
 if ruby -e "exit(%x{gem --version} < \"1.6.2\" ? 0 : -1 )" ; then
   echo -e "`date` \n\n**** \n**** Updating rubygems:\n****\n"
-  gem install --no-rdoc --no-ri rubygems-update --version=1.6.2
-  update_rubygems --version=1.6.2
+  # screw you Debian
+  REALLY_GEM_UPDATE_SYSTEM=1 gem update --system
+  # screw you rubygems
+  for foo in /usr/lib/ruby/site_ruby/*/rubygems/deprecate.rb ; do
+    # Don't have to be any such deprecations, in which case $foo won't exist
+    [ -f "$foo" ] && sudo sed -i.bak 's!@skip ||= false!true!' "$foo"
+  done
 fi
 
 echo -e "`date` \n\n**** \n**** Installing chef:\n****\n"
 gem install ohai --no-rdoc --no-ri
 gem install chef --no-rdoc --no-ri <%= bootstrap_version_string %>
-gem install      --no-rdoc --no-ri extlib json ruby-shadow right_aws
+# gems needed for the client.rb or so generically useful you want them at hand
+gem install      --no-rdoc --no-ri extlib bundler json right_aws pry
 
 else # no chef-client
 echo -e "`date` \n\n**** \n**** Chef is present -- skipping apt/ruby/chef installation\n****\n"
-fi 
+fi # end ruby+chef install
 
 # fix a bug in chef that prevents debugging template errors
-bad_template_file='/usr/lib/ruby/gems/1.9.2-p290/gems/chef-0.10.4/lib/chef/mixin/template.rb'
+# will not work with --prerelease but that's OK hopefully opscode patches this crap soon
+bad_template_file="/usr/lib/ruby/gems/${RUBY_VERSION}/gems/chef-${CHEF_VERSION}/lib/chef/mixin/template.rb"
 if  echo "0505c482b8b0b333ac71bbc8a1795d19  $bad_template_file" | md5sum -c - 2>/dev/null ; then
   curl https://github.com/mrflip/chef/commit/655a1967253a8759afb54f30b818bbcb7c309198.patch | sudo patch $bad_template_file
 fi
@@ -87,7 +104,7 @@ awk NF /tmp/knife-bootstrap/validation.pem > /etc/chef/validation.pem
 echo -e "`date` \n\n**** \n**** Nuking our temp files:\n****\n"
 
 cd /tmp
-# rm -rf /tmp/knife-bootstrap
+rm -rf /tmp/knife-bootstrap
 
 echo -e "`date` \n\n**** \n**** Creating chef client script:\n****\n"
 
@@ -105,21 +122,21 @@ EOP
 ) > /etc/chef/first-boot.json
 
 echo -e "`date` \n\n**** \n**** Adding chef client runit scripts:\n****\n"
-service chef-client stop >/dev/null 2>&1 ; sleep 1 ; killall chef-client 2>/dev/null ; true
+( service chef-client stop >/dev/null 2>&1 ; sleep 1 ; killall chef-client 2>/dev/null ) || true
 mkdir -p /var/log/chef /var/chef /etc/service /etc/sv/chef-client/{log/main,supervise} 
-cat > /etc/sv/chef-client/log/run <<EOF
+cat > /etc/sv/chef-client/log/run <<EOP
 #!/bin/bash
 exec svlogd -tt ./main
-EOF
-cat > /etc/sv/chef-client/run <<EOF
+EOP
+cat > /etc/sv/chef-client/run <<EOP
 #!/bin/bash
 exec 2>&1
 exec /usr/bin/env chef-client -i 43200 -s 20 -L /var/log/chef/client.log
-EOF
+EOP
 chmod +x  /etc/sv/chef-client/log/run /etc/sv/chef-client/run
 ln -nfs /usr/bin/sv /etc/init.d/chef-client
 
-service chef-client stop ; true
+service chef-client stop >/dev/null 2>&1 || true
 
 <%- if (@config[:bootstrap_runs_chef_client].to_s == 'true') || (@chef_config.knife[:bootstrap_runs_chef_client].to_s == 'true') %>
 echo -e "`date` \n\n**** \n**** First run of chef:\n****\n"
@@ -129,6 +146,7 @@ set +e
 <%- end %>
 
 echo -e "`date` \n\n**** \n**** Cleanup:\n****\n"
+# make locate work good
 updatedb
 
 echo -e "`date` \n\n**** \n**** Enabling chef client service:\n****\n"
@@ -136,4 +154,4 @@ ln -nfs /etc/sv/chef-client /etc/service/chef-client
 service chef-client start
 
 echo -e "`date` \n\n**** \n**** Cluster Chef client bootstrap complete\n****\n"
-'
+EOF

data/lib/chef/knife/bootstrap/ubuntu11.10-cluster_chef.erb

@@ -1,8 +1,9 @@
 bash <<EOF || echo "Chef bootstrap failed!"
 
-# This is the ubuntu-10.04-cluster_chef script from infochimps, which
-# is based on the ubuntu-10.04-gems script from opscode, but it
-# * installs ruby 1.9.2, not 1.8.7 using the system ruby
+# This is the ubuntu oneiric bootstrap script from infochimps' cluster_chef. It is
+# based on the opscode bootstrap script, with the important differences being it:
+#
+# * installs ruby 1.9.2 (not 1.8.7) using the system ruby
 # * upgrades rubygems rather than installing from source
 # * pushes the node identity into the first-boot.json
 # * installs the chef-client service and kicks off the first run of chef
@@ -11,6 +12,9 @@ set -e
 
 <%= (@config[:verbosity].to_i > 1 ? 'set -v' : '') %>
 
+RUBY_VERSION=1.9.1
+CHEF_VERSION=<%= bootstrap_version_string.gsub(/.*[\s=]/,"") %>
+
 mkdir -p /tmp/knife-bootstrap
 chmod 700 /tmp/knife-bootstrap 
 cd /tmp/knife-bootstrap
@@ -19,26 +23,31 @@ cd /tmp/knife-bootstrap
 eval `cat /etc/lsb-release `
 export DEBIAN_FRONTEND=noninteractive
 
+date > /etc/box_build_time
+
 # source for sun java if you want to install it later
 apt-get install -y python-software-properties
 add-apt-repository -y ppa:ferramroberto/java
-apt-get -y update
 
-if [ ! -f /usr/bin/chef-client ]; then
 echo -e "`date` \n\n**** \n**** apt update:\n****\n"
-apt-get    update
+apt-get -y update
 apt-get -y upgrade
 
 echo -e "`date` \n\n**** \n**** Installing base packages:\n****\n"
-apt-get install -y build-essential wget ruby1.9.1 ruby1.9.1-dev runit zlib1g-dev libssl-dev openssl libcurl4-openssl-dev libreadline6-dev libyaml-dev
+apt-get -y install build-essential make wget curl runit zlib1g-dev libssl-dev openssl libcurl4-openssl-dev libxml2-dev libxslt-dev libyaml-dev libreadline6 libreadline6-dev
+apt-get clean
+
+if [ ! -f /usr/bin/chef-client ]; then
+echo -e "`date` \n\n**** \n**** Installing ruby version ${RUBY_VERSION}:\n****\n"
+
+apt-get install -y ruby1.9.1 ruby1.9.1-dev
 
 if ruby -e "exit(%x{gem --version} < \"1.6.2\" ? 0 : -1 )" ; then
   echo -e "`date` \n\n**** \n**** Updating rubygems:\n****\n"
   # screw you Debian
   REALLY_GEM_UPDATE_SYSTEM=1 gem update --system
   # screw you rubygems
-  for foo in /usr/lib/ruby/site_ruby/*/rubygems/deprecate.rb
-  do
+  for foo in /usr/lib/ruby/site_ruby/*/rubygems/deprecate.rb ; do
     # Don't have to be any such deprecations, in which case $foo won't exist
     [ -f "$foo" ] && sudo sed -i.bak 's!@skip ||= false!true!' "$foo"
   done
@@ -47,14 +56,16 @@ fi
 echo -e "`date` \n\n**** \n**** Installing chef:\n****\n"
 gem install ohai --no-rdoc --no-ri
 gem install chef --no-rdoc --no-ri <%= bootstrap_version_string %>
-gem install      --no-rdoc --no-ri bundler pry cheat
+# gems needed for the client.rb or so generically useful you want them at hand
+gem install      --no-rdoc --no-ri extlib bundler json right_aws pry
 
 else # no chef-client
 echo -e "`date` \n\n**** \n**** Chef is present -- skipping apt/ruby/chef installation\n****\n"
-fi 
+fi # end ruby+chef install
 
 # fix a bug in chef that prevents debugging template errors
-bad_template_file='/usr/lib/ruby/gems/1.9.1/gems/chef-0.10.4/lib/chef/mixin/template.rb'
+# will not work with --prerelease but that's OK hopefully opscode patches this crap soon
+bad_template_file="/usr/lib/ruby/gems/${RUBY_VERSION}/gems/chef-${CHEF_VERSION}/lib/chef/mixin/template.rb"
 if  echo "0505c482b8b0b333ac71bbc8a1795d19  $bad_template_file" | md5sum -c - 2>/dev/null ; then
   curl https://github.com/mrflip/chef/commit/655a1967253a8759afb54f30b818bbcb7c309198.patch | sudo patch $bad_template_file
 fi
@@ -81,7 +92,7 @@ awk NF /tmp/knife-bootstrap/validation.pem > /etc/chef/validation.pem
 echo -e "`date` \n\n**** \n**** Nuking our temp files:\n****\n"
 
 cd /tmp
-# rm -rf /tmp/knife-bootstrap
+rm -rf /tmp/knife-bootstrap
 
 echo -e "`date` \n\n**** \n**** Creating chef client script:\n****\n"
 
@@ -99,7 +110,7 @@ EOP
 ) > /etc/chef/first-boot.json
 
 echo -e "`date` \n\n**** \n**** Adding chef client runit scripts:\n****\n"
-(service chef-client stop >/dev/null 2>&1 ; sleep 1 ; killall chef-client 2>/dev/null) || true
+( service chef-client stop >/dev/null 2>&1 ; sleep 1 ; killall chef-client 2>/dev/null ) || true
 mkdir -p /var/log/chef /var/chef /etc/service /etc/sv/chef-client/{log/main,supervise} 
 cat > /etc/sv/chef-client/log/run <<EOP
 #!/bin/bash
@@ -123,6 +134,7 @@ set +e
 <%- end %>
 
 echo -e "`date` \n\n**** \n**** Cleanup:\n****\n"
+# make locate work good
 updatedb
 
 echo -e "`date` \n\n**** \n**** Enabling chef client service:\n****\n"

data/lib/chef/knife/cluster_kick.rb

@@ -48,6 +48,9 @@ class Chef
 set -e
 <%= ((config[:verbosity].to_i > 1) ? "set -v" : "") %>
 
+if sudo service chef-client status ; then
+
+# running
 pid_file="<%= config[:pid_file] %>"
 log_file=/var/log/chef/client.log
 
@@ -71,6 +74,11 @@ sudo true
 pid="$(sudo cat $pid_file)"
 sudo kill -USR1 "$pid"
 sed -r "/(ERROR: Sleeping for [0-9]+ seconds before trying again|INFO: Report handlers complete)\$/{q}" $pipe
+
+else
+  echo -e "****\n\nchef-client daemon not running, invoking chef-client directly\n\n****\n"
+  sudo chef-client
+fi
 EOF
       end
 

data/lib/chef/knife/cluster_kill.rb

@@ -30,12 +30,12 @@ class Chef
         :default     => false
       option :cloud,
         :long        => '--[no-]cloud',
-        :description => "Delete machines from cloud (default is to delete, use --no-cloud to skip)",
+        :description => "Kill machines from cloud (default is yes, terminate machines; use --no-cloud to skip)",
         :boolean     => true,
         :default     => true
       option :chef,
         :long        => "--[no-]chef",
-        :description => "Delete the chef node and client (default is to delete, use --no-chef to skip)",
+        :description => "Delete the chef node and client (default is yes, delete chef objects; use --no-chef to skip)",
         :boolean     => true,
         :default     => true
 

data/lib/chef/knife/cluster_proxy.rb

@@ -99,22 +99,28 @@ class Chef
         pac_filename = File.expand_path(File.join('/tmp', "cluster_chef_proxy-#{ENV['USER']}.pac"))
         ui.info("point your browser at PAC (automatic proxy config file) file://#{pac_filename}")
         File.open(pac_filename, 'w') do |f|
-          f.print %Q{function FindProxyForURL(url, host) {
-  if ((shExpMatch(host, "*ec2*.amazonaws.com"      )) ||
-      (shExpMatch(host, "*ec2.internal*"           )) ||
-      (shExpMatch(host, "*compute-*.amazonaws.com" )) ||
-      (shExpMatch(host, "*compute-*.internal*"     )) ||
-      (shExpMatch(host, "*domu*.internal*"         )) ||
-      (shExpMatch(host, "10.*"                     ))
+          f.print proxy_pac_contents
+        end
+      end
+
+      EC2_PROXY_PATTERNS = [ "*ec2*.amazonaws.com", "*ec2.internal*", "*compute-*.amazonaws.com", "*compute-*.internal*", "*domu*.internal*", "10.*",]
+
+      def proxy_pac_contents
+        proxy_patterns  = EC2_PROXY_PATTERNS
+        proxy_patterns += Array(Chef::Config[:cluster_proxy_patterns])
+        rules = proxy_patterns.compact.map{|str| "(shExpMatch(host, %-28s))" % %Q{"#{str}"} }
+        %Q{function FindProxyForURL(url, host) {
+  if (#{rules.join(" ||\n      ")}
       ) {
     return "SOCKS5 localhost:#{config[:socks_port]}";
   }
   return "DIRECT";
-}
-         }
-        end
+}\n}
       end
 
+
+
+
     end
   end
 end

data/lib/chef/knife/cluster_show.rb

@@ -27,6 +27,11 @@ class Chef
       end
 
       banner "knife cluster show CLUSTER_NAME [FACET_NAME [INDEXES]] (options)"
+      option :cloud,
+        :long        => "--[no-]cloud",
+        :description => "Look up machines on AWS cloud (default is yes, look up machines; use --no-cloud to skip)",
+        :default     => true,
+        :boolean     => true
 
       def run
         load_cluster_chef